source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/VC_HDF5_VSANS_Utils.ipf @ 969

Last change on this file since 969 was 969, checked in by srkline, 7 years ago

more additions to the R/W functions to access VSANS data files

File size: 16.4 KB
Line 
1#pragma rtGlobals=1             // Use modern global access method.
2
3//
4// This file has utility procedures to be able to read/write
5// test HDF5 (from Igor) for SANS and VSANS in Nexus format
6//
7// It is predominantly to be able to WRITE a full Nexus file from Igor
8// -- this is only necessary to be able to write out "fake" VSANS files from VCALC
9//    since I need a "template" of the Nexus folder structure to start from and fill in
10//
11// It doesn't reproduce the NICE logs, but will leave a spece for them if
12// it is read in and is part of the xref.
13//
14// It may be easier to hard-wire my own folder definition (NewDataFolder)
15// that is MY Nexus file to write out and use that as the base case
16// rather than rely on Pete's code, which is terribly slow for writing.
17// Attributes are the issue, but it may be unimportant if they are missing.
18//
19
20
21
22//
23// There's an odd sequence of steps that need to be done to use HDFGateway
24// to most easily write out attributes, and be able to read then back in.
25// Rather awkward, but it works without me needing to go nuts with an atomistic
26// understanding of HDF5. This is necessary for me to be able to write out
27// Nexus files, which is important both for testing, and for simulation.
28//
29// This is done as a set of macros that need to be applied in a specific sequence
30// in order to be able to generate HDF5___xref
31// Ideally, the HDF5___xref wave can be saved/imported to make proper saving possible
32// once the "tree" is finalized.
33//
34//
35// The basic saving routines are here, just as Proc rather than Macro menu clutter
36//
37
38
39// Next... think of all of R/W access needed
40//
41// the simple read/write works...
42// linear_data does not seem to need to be transposed at all
43//
44//  -- this seems too easy. what am I doing wrong? Is something getting garbled when I
45// write back any single values back to the file
46//
47// -- try a string value next
48// -- then start to write the generic get/write functions
49//
50
51//
52// do I ditch the RealsRead/IntegersRead/TextRead? It makes little sense now.
53// maybe copy a "dataInfo" folder/subfolders. can't keep them all (bloat)
54// but then what about multiple files added together?
55//
56
57
58Macro Load_Nexus_V_Template()
59        H_HDF5Gate_Read_Raw("")
60        String tmpStr=root:file_name  //SRK - so I can get the file name that was loaded
61       
62// this is the folder name
63        String base_name = StringFromList(0,File_Name,".")
64        RenameDataFolder $("root:"+base_name), V_Nexus_Template
65End
66
67
68Macro Fill_Nexus_V_Template()
69        H_Fill_VSANS_Template_wSim()
70End
71
72
73Macro Save_Nexus_V_Template()
74        H_HDF5Gate_Write_Raw("root:V_Nexus_Template:","")
75End
76
77
78
79
80
81
82
83
84
85
86
87
88
89/////////////////////////////
90
91/////////////below is largely depricated, ugly dance to be able to "fake" a file from Igor
92// which was not complete anyways.
93
94
95
96Macro IgorOnly_Setup_VSANS_Struct()
97
98        // lays out the tree and fills with dummy values
99        H_Setup_VSANS_Structure()
100       
101        // writes in the attributes
102        H_Fill_VSANS_Attributes()
103       
104        // fill in with VCALC simulation bits
105        H_Fill_VSANS_wSim()
106       
107End
108
109Macro IgorOnly_Save_VSANS_Nexus(fileName)
110        String fileName="Test_VSANS_file"
111
112        // save as HDF5 (no attributes saved yet)
113        Save_VSANS_file("root:VSANS_file", fileName+".h5")
114       
115        // read in a data file using the gateway-- reads from the home path
116        H_HDF5Gate_Read_Raw(fileName+".h5")
117       
118        // after reading in a "partial" file using the gateway (to generate the xref)
119        // Save the xref to disk (for later use)
120        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
121       
122        // after you've generated the HDF5___xref, load it in and copy it
123        // to the necessary folder location.
124        Copy_HDF5___xref("root:VSANS_file", "HDF5___xref")
125       
126        // writes out the contents of a data folder using the gateway
127        H_HDF5Gate_Write_Raw("root:VSANS_file", fileName+".h5")
128
129        // re-load the data file using the gateway-- reads from the home path
130        // now with attributes
131        H_HDF5Gate_Read_Raw(fileName+".h5")
132       
133End
134
135
136
137Macro IgorOnly_Setup_SANS_Struct()
138
139        // lays out the tree and fills with dummy values
140        H_Setup_SANS_Structure()
141       
142        // writes in the attributes
143        H_Fill_SANS_Attributes()
144       
145        // fill in with VCALC simulation bits
146        H_Fill_SANS_wSim()
147
148End
149
150Macro IgorOnly_Save_SANS_Nexus(fileName)
151        String fileName="Test_SANS_file"
152       
153        // save as HDF5 (no attributes saved yet) (save_VSANS is actually generic HDF...)
154        Save_VSANS_file("root:SANS_file", fileName+".h5")
155       
156        // read in a data file using the gateway-- reads from the home path
157        H_HDF5Gate_Read_Raw(fileName+".h5")
158       
159        // after reading in a "partial" file using the gateway (to generate the xref)
160        // Save the xref to disk (for later use)
161        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
162       
163        // after you've generated the HDF5___xref, load it in and copy it
164        // to the necessary folder location.
165        Copy_HDF5___xref("root:SANS_file", "HDF5___xref")
166       
167        // writes out the contents of a data folder using the gateway
168        H_HDF5Gate_Write_Raw("root:SANS_file", fileName+".h5")
169
170        // re-load the data file using the gateway-- reads from the home path
171        // now with attributes
172        H_HDF5Gate_Read_Raw(fileName+".h5")
173
174
175End
176
177
178// TODO
179// currently, there are no dummy fill values or attributes for the fake DIV file
180//
181Macro Setup_VSANS_DIV_Struct()
182
183        // lays out the tree and fills with dummy values
184        H_Setup_VSANS_DIV_Structure()
185       
186        // writes in the attributes
187//      H_Fill_VSANS_Attributes()
188       
189        // fill in with VCALC simulation bits
190//      H_Fill_VSANS_wSim()
191       
192End
193
194Macro Save_VSANS_DIV_Nexus(fileName)
195        String fileName="Test_VSANS_DIV_file"
196
197        // save as HDF5 (no attributes saved yet)
198        Save_VSANS_file("root:VSANS_DIV_file", fileName+".h5")
199       
200        // read in a data file using the gateway-- reads from the home path
201        H_HDF5Gate_Read_Raw(fileName+".h5")
202       
203        // after reading in a "partial" file using the gateway (to generate the xref)
204        // Save the xref to disk (for later use)
205        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
206       
207        // after you've generated the HDF5___xref, load it in and copy it
208        // to the necessary folder location.
209        Copy_HDF5___xref("root:VSANS_DIV_file", "HDF5___xref")
210       
211        // writes out the contents of a data folder using the gateway
212        H_HDF5Gate_Write_Raw("root:VSANS_DIV_file", fileName+".h5")
213
214        // re-load the data file using the gateway-- reads from the home path
215        // now with attributes
216        H_HDF5Gate_Read_Raw(fileName+".h5")
217       
218End
219
220
221
222
223
224
225
226
227
228
229
230
231
232//
233// saves a specified folder, with a given filename.
234// saves to the home path
235//
236Proc Save_VSANS_file(dfPath, filename)
237        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
238        String filename = "Test_VSANS_file.h5"
239       
240        H_NXSANS_SaveGroupAsHDF5(dfPath, filename)
241End
242
243
244//     
245// this is my procedure to save the folders to HDF5, once I've filled the folder tree
246//
247// this does NOT save attributes, but gets the folder structure correct
248//
249Function H_NXSANS_SaveGroupAsHDF5(dfPath, filename)
250        String dfPath   // e.g., "root:FolderA" or ":"
251        String filename
252
253        Variable result = 0     // 0 means no error
254       
255        Variable fileID
256        HDF5CreateFile/P=home /O /Z fileID as filename
257        if (V_flag != 0)
258                Print "HDF5CreateFile failed"
259                return -1
260        endif
261
262        HDF5SaveGroup /IGOR=0 /O /R /Z $dfPath, fileID, "."
263//      HDF5SaveGroup /O /R /Z $dfPath, fileID, "."
264        if (V_flag != 0)
265                Print "HDF5SaveGroup failed"
266                result = -1
267        endif
268       
269        HDF5CloseFile fileID
270
271        return result
272End
273
274// passing null file string presents a dialog
275// these two procedures will use the full Xrefs, so they can write out full
276// files. They are, however, slow
277Proc Read_Nexus_Xref()
278        H_HDF5Gate_Read_Raw("")
279End
280
281Proc Write_Nexus_Xref()
282        H_HDF5Gate_Write_Raw()
283End
284
285//
286// writes out the contents of a data folder using the gateway
287// -- the HDF5___xref wave must be present at the top level for it to
288//    write out anything.
289//
290Proc H_HDF5Gate_Write_Raw(dfPath,filename)
291        String dfPath   = "root:"       // e.g., "root:FolderA" or ":"
292        String filename = "Test_VSANS_file.h5"
293//      Prompt dfPath, "Data Folder",popup,H_GetAList(4)       
294//      Prompt fileName, "File name"
295       
296//      DoPrompt "Folder and file to write", dfPath,fileName
297        // Check our work so far.
298        // If something prints, there was an error above.
299        print H5GW_ValidateFolder(dfPath)
300
301        print H5GW_WriteHDF5(dfPath, filename)
302       
303        SetDataFolder root:
304End
305
306//
307// read in a data file using the gateway
308//
309// reads from the home path
310//
311Proc H_HDF5Gate_Read_Raw(file)
312        String file
313       
314//      NewDataFolder/O/S root:newdata
315        Print H5GW_ReadHDF5("", file)   // reads into current folder
316        SetDataFolder root:
317End
318
319
320
321// this is in PlotUtils, but duplicated here
322// utility used in the "PlotSmeared...() macros to get a list of data folders
323//
324//1:    Waves.
325//2:    Numeric variables.
326//3:    String variables.
327//4:    Data folders.
328Function/S H_GetAList(type)
329        Variable type
330       
331        SetDataFolder root:
332       
333        String objName,str=""
334        Variable index = 0
335        do
336                objName = GetIndexedObjName(":", type, index)
337                if (strlen(objName) == 0)
338                        break
339                endif
340                //Print objName
341                str += objName + ";"
342                index += 1
343        while(1)
344       
345        // remove myGlobals, Packages, etc. from the folder list
346        if(type==4)
347                str = RemoveFromList("myGlobals", str , ";" )
348                str = RemoveFromList("Packages", str, ";")
349        endif
350       
351        return(str)
352End
353
354//
355// after reading in a "partial" file using the gateway (to generate the xref)
356// Save the xref to disk (for later use)
357//
358Proc Save_HDF5___xref(dfPath, filename)
359        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
360        String filename = "HDF5___xref"
361
362        Save/T/P=home $(dfPath+":HDF5___xref") as "HDF5___xref.itx"
363
364//      Copy_HDF5___xref(dfPath, filename)
365       
366        SetDataFolder root:
367End
368
369//
370// after you've generated the HDF5___xref, load it in and copy it to
371// the necessary folder location.
372// - then all is set to *really* write out the file correctly, including the attributes
373//
374Proc Copy_HDF5___xref(dfPath, filename)
375        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
376        String filename = "HDF5___xref"
377
378        if(exists(filename) != 1)
379                //load it in   
380                LoadWave/T/P=home/O "HDF5___xref.itx"
381        endif
382       
383        Duplicate/O HDF5___xref, $(dfPath+":HDF5___xref")
384       
385        SetDataFolder root:
386End
387
388
389
390//////// testing procedures, may be of use, maybe not //////////////
391
392
393//////// Two procedures that test out Pete Jemain's HDF5Gateway
394//
395// This works fine, but it may not be terribly compatible with the way NICE will eventually
396// write out the data files. I'll have very little control over that and I'll need to cobble together
397// a bunch of fixes to cover up their mistakes.
398//
399// Using Nick Hauser's code as a starting point may be a lot more painful, but more flexible in the end.
400//
401// I'm completely baffled about what to do with attributes. Are they needed, is this the best way to deal
402// with them, do I care about reading them in, and if I do, why?
403//
404Proc H_HDF5Gate_WriteTest()
405
406        // create the folder structure
407        NewDataFolder/O/S root:mydata
408        NewDataFolder/O sasentry
409        NewDataFolder/O :sasentry:sasdata
410
411        // create the waves
412        Make/O :sasentry:sasdata:I0
413        Make/O :sasentry:sasdata:Q0
414
415        Make/O/N=0 Igor___folder_attributes
416        Make/O/N=0 :sasentry:Igor___folder_attributes
417        Make/O/N=0 :sasentry:sasdata:Igor___folder_attributes
418
419        // create the attributes
420        Note/K Igor___folder_attributes, "producer=IgorPro\rNX_class=NXroot"
421        Note/K :sasentry:Igor___folder_attributes, "NX_class=NXentry"
422        Note/K :sasentry:sasdata:Igor___folder_attributes, "NX_class=NXdata"
423        Note/K :sasentry:sasdata:I0, "units=1/cm\rsignal=1\rtitle=reduced intensity"
424        Note/K :sasentry:sasdata:Q0, "units=1/A\rtitle=|scattering vector|"
425
426        // create the cross-reference mapping
427        Make/O/T/N=(5,2) HDF5___xref
428        Edit/K=0 'HDF5___xref';DelayUpdate
429        HDF5___xref[0][1] = ":"
430        HDF5___xref[1][1] = ":sasentry"
431        HDF5___xref[2][1] = ":sasentry:sasdata"
432        HDF5___xref[3][1] = ":sasentry:sasdata:I0"
433        HDF5___xref[4][1] = ":sasentry:sasdata:Q0"
434        HDF5___xref[0][0] = "/"
435        HDF5___xref[1][0] = "/sasentry"
436        HDF5___xref[2][0] = "/sasentry/sasdata"
437        HDF5___xref[3][0] = "/sasentry/sasdata/I"
438        HDF5___xref[4][0] = "/sasentry/sasdata/Q"
439
440        // Check our work so far.
441        // If something prints, there was an error above.
442        print H5GW_ValidateFolder("root:mydata")
443
444        // set I0 and Q0 to your data
445
446        print H5GW_WriteHDF5("root:mydata", "mydata.h5")
447       
448        SetDataFolder root:
449End
450
451
452//
453// given a filename of a SANS data filename of the form
454// name.anything
455// returns the name as a string without the ".fbdfasga" extension
456//
457// returns the input string if a "." can't be found (maybe it wasn't there)
458//
459Function/S H_RemoveDotExtension(item)
460        String item
461        String invalid = item   //
462        Variable num=-1
463       
464        //find the "dot"
465        String runStr=""
466        Variable pos = strsearch(item,".",0)
467        if(pos == -1)
468                //"dot" not found
469                return (invalid)
470        else
471                //found, get all of the characters preceeding it
472                runStr = item[0,pos-1]
473                return (runStr)
474        Endif
475End
476
477
478//
479// Writing attributes to a group and to a dataset.
480// example from the WM documentation for HDF5SaveData
481//
482Function DemoAttributes(w)
483        Wave w
484
485        Variable result = 0     // 0 means no error
486       
487        // Create file
488        Variable fileID
489        HDF5CreateFile/P=home /O /Z fileID as "Test.h5"
490        if (V_flag != 0)
491                Print "HDF5CreateFile failed"
492                return -1
493        endif
494
495        // Write an attribute to the root group
496        Make /FREE /T /N=1 groupAttribute = "This is a group attribute"
497        HDF5SaveData /A="GroupAttribute" groupAttribute, fileID, "/"
498       
499        // Save wave as dataset
500        HDF5SaveData /O /Z w, fileID    // Uses wave name as dataset name
501        if (V_flag != 0)
502                Print "HDF5SaveData failed"
503                result = -1
504        endif
505
506        // Write an attribute to the dataset
507        Make /FREE /T /N=1 datasetAttribute = "This is a dataset attribute"
508        String datasetName = NameOfWave(w)
509        HDF5SaveData /A="DatasetAttribute" datasetAttribute, fileID, datasetName
510
511        HDF5CloseFile fileID
512       
513        return result
514End     // The attribute waves are automatically killed since they are free waves
515
516
517
518//     
519//Function H_Test_HDFWriteTrans(fname,val)
520//      String fname
521//      Variable val
522//     
523//     
524//      String str
525//      PathInfo home
526//      str = S_path
527//     
528//      H_WriteTransmissionToHeader(str+fname,val)
529//     
530//      return(0)
531//End
532//
533//Function H_WriteTransmissionToHeader(fname,trans)
534//      String fname
535//      Variable trans
536//     
537//      Make/O/D/N=1 wTmpWrite
538//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
539//      String varName = "TRNS"
540//      wTmpWrite[0] = trans //
541//
542//      variable err
543//      err = HDFWrite_Wave(fname, groupName, varName, wTmpWrite)
544//      KillWaves wTmpWrite
545//     
546//      //err not handled here
547//             
548//      return(0)
549//End
550
551
552
553Function H_Test_ListAttributes(fname,groupName)
554        String fname,groupName
555        Variable trans
556       
557//      Make/O/D/N=1 wTmpWrite
558//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
559//      String varName = "TRNS"
560//      wTmpWrite[0] = trans //
561        String str
562        PathInfo home
563        str = S_path
564       
565        variable err
566        err = H_HDF_ListAttributes(str+fname, groupName)
567       
568        //err not handled here
569               
570        return(0)
571End
572
573Function H_HDF_ListAttributes(fname, groupName)
574        String fname, groupName
575       
576        variable err=0, fileID,groupID
577        String cDF = getDataFolder(1), temp
578        String NXentry_name, attrValue=""
579       
580        STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
581        InitHDF5DataInfo(di)    // Initialize structure.
582       
583        try     
584                HDF5OpenFile /Z fileID  as fname  //open file read-write
585                if(!fileID)
586                        err = 1
587                        abort "HDF5 file does not exist"
588                endif
589               
590                HDF5OpenGroup /Z fileID , groupName, groupID
591
592        //      (QUOKKA) !! At the moment, there is no entry for sample thickness in our data file
593        //      therefore create new HDF5 group to enable write / patch command
594        //      comment out the following group creation once thickness appears in revised file
595       
596                if(!groupID)
597                        HDF5CreateGroup /Z fileID, groupName, groupID
598                        //err = 1
599                        //abort "HDF5 group does not exist"
600                else
601
602//                      HDF5AttributeInfo(fileID, "/", 1, "file_name", 0, di)
603                        HDF5AttributeInfo(fileID, "/", 1, "NeXus_version", 0, di)
604                        Print di
605
606//                      see the HDF5 Browser  for how to get the actual <value> of the attribute. See GetPreviewString in
607//        or in FillGroupAttributesList or in FillDatasetAttributesList (from FillLists)
608//                      it seems to be ridiculously complex to get such a simple bit of information - the HDF5BrowserData STRUCT
609//                      needs to be filled first. Ugh.
610                        attrValue = GetPreviewString(fileID, 1, di, "/entry", "cucumber")
611                        Print "attrValue = ",attrValue
612                       
613                       
614                        //get attributes and save them
615                        HDF5ListAttributes/TYPE=1 /Z fileID, groupName          //TYPE=1 means that we're referencing a group, not a dataset
616                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
617                       
618                        // passing the groupID works too, then the group name is not needed                     
619                        HDF5ListAttributes/TYPE=1 /Z groupID, "."               //TYPE=1 means that we're referencing a group, not a dataset
620                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
621                endif
622        catch
623
624                // catch any aborts here
625               
626        endtry
627       
628        if(groupID)
629                HDF5CloseGroup /Z groupID
630        endif
631       
632        if(fileID)
633                HDF5CloseFile /Z fileID
634        endif
635
636        setDataFolder $cDF
637        return err
638end
639
640
641/////////////// end of the testing procedures ////////////////
Note: See TracBrowser for help on using the repository browser.