source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/VC_HDF5_VSANS_Utils.ipf @ 1019

Last change on this file since 1019 was 1019, checked in by srkline, 6 years ago

adding procedures for:

simple save of a DIV file. no functionality to generate a DIV file yet, since I don't know how this will happen.

Simple dump of the file structure in a data "tree"

Verified that the error bars on the I(q) data are correctly calculated as standard error of the mean. There was never an issue with this, or with SANS calculations.

Started filling in "Correct" routines based on the SANS version. Only stubs present currently.

File size: 15.3 KB
Line 
1#pragma rtGlobals=1             // Use modern global access method.
2
3//
4// ********
5// TODO -- figure out how much of this file is really used, and how much is
6//    simply garbage now. I make "fake" data files by starting with a real data file
7//    written from NICE, and then I re-write proper values and data arrays to it from
8//    simulations in VCALC. So I do not need to define the structure myself.
9//
10//   JAN 2017
11//
12//
13//
14//
15//
16
17//
18// This file has utility procedures to be able to read/write
19// test HDF5 (from Igor) for SANS and VSANS in Nexus format
20//
21// It is predominantly to be able to WRITE a full Nexus file from Igor
22// -- this is only necessary to be able to write out "fake" VSANS files from VCALC
23//    since I need a "template" of the Nexus folder structure to start from and fill in
24//
25// It doesn't reproduce the NICE logs, but will leave a spece for them if
26// it is read in and is part of the xref.
27//
28// It may be easier to hard-wire my own folder definition (NewDataFolder)
29// that is MY Nexus file to write out and use that as the base case
30// rather than rely on Pete's code, which is terribly slow for writing.
31// Attributes are the issue, but it may be unimportant if they are missing.
32//
33
34
35
36//
37// There's an odd sequence of steps that need to be done to use HDFGateway
38// to most easily write out attributes, and be able to read then back in.
39// Rather awkward, but it works without me needing to go nuts with an atomistic
40// understanding of HDF5. This is necessary for me to be able to write out
41// Nexus files, which is important both for testing, and for simulation.
42//
43// This is done as a set of macros that need to be applied in a specific sequence
44// in order to be able to generate HDF5___xref
45// Ideally, the HDF5___xref wave can be saved/imported to make proper saving possible
46// once the "tree" is finalized.
47//
48//
49// The basic saving routines are here, just as Proc rather than Macro menu clutter
50//
51
52
53// Next... think of all of R/W access needed
54//
55// the simple read/write works...
56// linear_data does not seem to need to be transposed at all
57//
58//  -- this seems too easy. what am I doing wrong? Is something getting garbled when I
59// write back any single values back to the file
60//
61// -- try a string value next
62// -- then start to write the generic get/write functions
63//
64
65//
66// do I ditch the RealsRead/IntegersRead/TextRead? It makes little sense now.
67// maybe copy a "dataInfo" folder/subfolders. can't keep them all (bloat)
68// but then what about multiple files added together?
69//
70
71Menu "VSANS"
72        SubMenu "Nexus File Testing"
73                "Fill_Nexus_V_Template"
74                "Save_Nexus_V_Template"
75                "Load_Nexus_V_Template"
76                "-"
77                "IgorOnly_Setup_VSANS_Struct"
78                "IgorOnly_Save_VSANS_Nexus"
79                "IgorOnly_Setup_SANS_Struct"
80                "IgorOnly_Save_SANS_Struct"
81                "Setup_VSANS_DIV_Struct"
82                "Save_VSANS_DIV_Nexus"
83                "Setup_VSANS_MASK_Struct"
84                "Save_VSANS_MASK_Nexus"
85                "-"
86                "Read_Nexus with attributes",Read_Nexus_Xref()          //this will read with attributes
87                "Write_Nexus with attributes",Write_Nexus_Xref()                                //this will write out with attributes if read in by Read_Nexus_Xref
88        End
89End
90
91
92
93
94
95Proc Load_Nexus_V_Template()
96        H_HDF5Gate_Read_Raw("")
97        String tmpStr=root:file_name  //SRK - so I can get the file name that was loaded
98       
99// this is the folder name
100        String base_name = StringFromList(0,File_Name,".")
101        RenameDataFolder $("root:"+base_name), V_Nexus_Template
102End
103
104
105Proc Fill_Nexus_V_Template()
106        H_Fill_VSANS_Template_wSim()
107End
108
109
110Proc Save_Nexus_V_Template()
111        H_HDF5Gate_Write_Raw("root:V_Nexus_Template:","")
112End
113
114
115
116
117
118
119
120
121
122
123
124
125
126/////////////////////////////
127
128/////////////below is largely depricated, ugly dance to be able to "fake" a file from Igor
129// which was not complete anyways.
130
131
132
133Proc IgorOnly_Setup_VSANS_Struct()
134
135        // lays out the tree and fills with dummy values
136        H_Setup_VSANS_Structure()
137       
138        // writes in the attributes
139        H_Fill_VSANS_Attributes()
140       
141        // fill in with VCALC simulation bits
142        H_Fill_VSANS_wSim()
143       
144End
145
146Proc IgorOnly_Save_VSANS_Nexus(fileName)
147        String fileName="Test_VSANS_file"
148
149        // save as HDF5 (no attributes saved yet)
150        Save_VSANS_file("root:VSANS_file", fileName+".h5")
151       
152        // read in a data file using the gateway-- reads from the home path
153        H_HDF5Gate_Read_Raw(fileName+".h5")
154       
155        // after reading in a "partial" file using the gateway (to generate the xref)
156        // Save the xref to disk (for later use)
157        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
158       
159        // after you've generated the HDF5___xref, load it in and copy it
160        // to the necessary folder location.
161        Copy_HDF5___xref("root:VSANS_file", "HDF5___xref")
162       
163        // writes out the contents of a data folder using the gateway
164        H_HDF5Gate_Write_Raw("root:VSANS_file", fileName+".h5")
165
166        // re-load the data file using the gateway-- reads from the home path
167        // now with attributes
168        H_HDF5Gate_Read_Raw(fileName+".h5")
169       
170End
171
172
173
174Proc IgorOnly_Setup_SANS_Struct()
175
176        // lays out the tree and fills with dummy values
177        H_Setup_SANS_Structure()
178       
179        // writes in the attributes
180        H_Fill_SANS_Attributes()
181       
182        // fill in with VCALC simulation bits
183        H_Fill_SANS_wSim()
184
185End
186
187Proc IgorOnly_Save_SANS_Nexus(fileName)
188        String fileName="Test_SANS_file"
189       
190        // save as HDF5 (no attributes saved yet) (save_VSANS is actually generic HDF...)
191        Save_VSANS_file("root:SANS_file", fileName+".h5")
192       
193        // read in a data file using the gateway-- reads from the home path
194        H_HDF5Gate_Read_Raw(fileName+".h5")
195       
196        // after reading in a "partial" file using the gateway (to generate the xref)
197        // Save the xref to disk (for later use)
198        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
199       
200        // after you've generated the HDF5___xref, load it in and copy it
201        // to the necessary folder location.
202        Copy_HDF5___xref("root:SANS_file", "HDF5___xref")
203       
204        // writes out the contents of a data folder using the gateway
205        H_HDF5Gate_Write_Raw("root:SANS_file", fileName+".h5")
206
207        // re-load the data file using the gateway-- reads from the home path
208        // now with attributes
209        H_HDF5Gate_Read_Raw(fileName+".h5")
210
211
212End
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232// passing null file string presents a dialog
233// these two procedures will use the full Xrefs, so they can write out full
234// files. They are, however, slow
235Proc Read_Nexus_Xref()
236        H_HDF5Gate_Read_Raw("")
237End
238
239Proc Write_Nexus_Xref()
240        H_HDF5Gate_Write_Raw()
241End
242
243//
244// writes out the contents of a data folder using the gateway
245// -- the HDF5___xref wave must be present at the top level for it to
246//    write out anything.
247//
248Proc H_HDF5Gate_Write_Raw(dfPath,filename)
249        String dfPath   = "root:"       // e.g., "root:FolderA" or ":"
250        String filename = "Test_VSANS_file.h5"
251//      Prompt dfPath, "Data Folder",popup,H_GetAList(4)       
252//      Prompt fileName, "File name"
253       
254//      DoPrompt "Folder and file to write", dfPath,fileName
255        // Check our work so far.
256        // If something prints, there was an error above.
257//      print H5GW_ValidateFolder(dfPath)
258
259// H5GW_WriteHDF5 calls the validate function, so no need to call it before
260        print H5GW_WriteHDF5(dfPath, filename)
261       
262        SetDataFolder root:
263End
264
265//
266// read in a data file using the gateway
267//
268// reads from the home path
269//
270Proc H_HDF5Gate_Read_Raw(file)
271        String file
272       
273//      NewDataFolder/O/S root:newdata
274        Print H5GW_ReadHDF5("", file)   // reads into current folder
275        SetDataFolder root:
276End
277
278
279
280// this is in PlotUtils, but duplicated here
281// utility used in the "PlotSmeared...() macros to get a list of data folders
282//
283//1:    Waves.
284//2:    Numeric variables.
285//3:    String variables.
286//4:    Data folders.
287Function/S H_GetAList(type)
288        Variable type
289       
290        SetDataFolder root:
291       
292        String objName,str=""
293        Variable index = 0
294        do
295                objName = GetIndexedObjName(":", type, index)
296                if (strlen(objName) == 0)
297                        break
298                endif
299                //Print objName
300                str += objName + ";"
301                index += 1
302        while(1)
303       
304        // remove myGlobals, Packages, etc. from the folder list
305        if(type==4)
306                str = RemoveFromList("myGlobals", str , ";" )
307                str = RemoveFromList("Packages", str, ";")
308        endif
309       
310        return(str)
311End
312
313//
314// after reading in a "partial" file using the gateway (to generate the xref)
315// Save the xref to disk (for later use)
316//
317Proc Save_HDF5___xref(dfPath, filename)
318        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
319        String filename = "HDF5___xref"
320
321        Save/T/P=home $(dfPath+":HDF5___xref") as "HDF5___xref.itx"
322
323//      Copy_HDF5___xref(dfPath, filename)
324       
325        SetDataFolder root:
326End
327
328//
329// after you've generated the HDF5___xref, load it in and copy it to
330// the necessary folder location.
331// - then all is set to *really* write out the file correctly, including the attributes
332//
333Proc Copy_HDF5___xref(dfPath, filename)
334        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
335        String filename = "HDF5___xref"
336
337        if(exists(filename) != 1)
338                //load it in   
339                LoadWave/T/P=home/O "HDF5___xref.itx"
340        endif
341       
342        Duplicate/O HDF5___xref, $(dfPath+":HDF5___xref")
343       
344        SetDataFolder root:
345End
346
347
348
349//////// testing procedures, may be of use, maybe not //////////////
350
351
352//////// Two procedures that test out Pete Jemain's HDF5Gateway
353//
354// This works fine, but it may not be terribly compatible with the way NICE will eventually
355// write out the data files. I'll have very little control over that and I'll need to cobble together
356// a bunch of fixes to cover up their mistakes.
357//
358// Using Nick Hauser's code as a starting point may be a lot more painful, but more flexible in the end.
359//
360// I'm completely baffled about what to do with attributes. Are they needed, is this the best way to deal
361// with them, do I care about reading them in, and if I do, why?
362//
363Proc H_HDF5Gate_WriteTest()
364
365        // create the folder structure
366        NewDataFolder/O/S root:mydata
367        NewDataFolder/O sasentry
368        NewDataFolder/O :sasentry:sasdata
369
370        // create the waves
371        Make/O :sasentry:sasdata:I0
372        Make/O :sasentry:sasdata:Q0
373
374        Make/O/N=0 Igor___folder_attributes
375        Make/O/N=0 :sasentry:Igor___folder_attributes
376        Make/O/N=0 :sasentry:sasdata:Igor___folder_attributes
377
378        // create the attributes
379        Note/K Igor___folder_attributes, "producer=IgorPro\rNX_class=NXroot"
380        Note/K :sasentry:Igor___folder_attributes, "NX_class=NXentry"
381        Note/K :sasentry:sasdata:Igor___folder_attributes, "NX_class=NXdata"
382        Note/K :sasentry:sasdata:I0, "units=1/cm\rsignal=1\rtitle=reduced intensity"
383        Note/K :sasentry:sasdata:Q0, "units=1/A\rtitle=|scattering vector|"
384
385        // create the cross-reference mapping
386        Make/O/T/N=(5,2) HDF5___xref
387        Edit/K=0 'HDF5___xref';DelayUpdate
388        HDF5___xref[0][1] = ":"
389        HDF5___xref[1][1] = ":sasentry"
390        HDF5___xref[2][1] = ":sasentry:sasdata"
391        HDF5___xref[3][1] = ":sasentry:sasdata:I0"
392        HDF5___xref[4][1] = ":sasentry:sasdata:Q0"
393        HDF5___xref[0][0] = "/"
394        HDF5___xref[1][0] = "/sasentry"
395        HDF5___xref[2][0] = "/sasentry/sasdata"
396        HDF5___xref[3][0] = "/sasentry/sasdata/I"
397        HDF5___xref[4][0] = "/sasentry/sasdata/Q"
398
399        // Check our work so far.
400        // If something prints, there was an error above.
401        print H5GW_ValidateFolder("root:mydata")
402
403        // set I0 and Q0 to your data
404
405        print H5GW_WriteHDF5("root:mydata", "mydata.h5")
406       
407        SetDataFolder root:
408End
409
410
411//
412// given a filename of a VSANS data file of the form
413// name.anything.any.other (any number of extensions is OK)
414// returns the name as a string without ANY ".fbdfasga" extension
415//
416// returns the input string if a "." can't be found (maybe it wasn't there)
417//
418Function/S H_RemoveDotExtension(item)
419        String item
420        String invalid = item   //
421        Variable num=-1
422       
423        //find the "dot"
424        String runStr=""
425        Variable pos = strsearch(item,".",0)
426        if(pos == -1)
427                //"dot" not found
428                return (invalid)
429        else
430                //found, get all of the characters preceeding it
431                runStr = item[0,pos-1]
432                return (runStr)
433        Endif
434End
435
436
437//
438// Writing attributes to a group and to a dataset.
439// example from the WM documentation for HDF5SaveData
440//
441Function DemoAttributes(w)
442        Wave w
443
444        Variable result = 0     // 0 means no error
445       
446        // Create file
447        Variable fileID
448        HDF5CreateFile/P=home /O /Z fileID as "Test.h5"
449        if (V_flag != 0)
450                Print "HDF5CreateFile failed"
451                return -1
452        endif
453
454        // Write an attribute to the root group
455        Make /FREE /T /N=1 groupAttribute = "This is a group attribute"
456        HDF5SaveData /A="GroupAttribute" groupAttribute, fileID, "/"
457       
458        // Save wave as dataset
459        HDF5SaveData /O /Z w, fileID    // Uses wave name as dataset name
460        if (V_flag != 0)
461                Print "HDF5SaveData failed"
462                result = -1
463        endif
464
465        // Write an attribute to the dataset
466        Make /FREE /T /N=1 datasetAttribute = "This is a dataset attribute"
467        String datasetName = NameOfWave(w)
468        HDF5SaveData /A="DatasetAttribute" datasetAttribute, fileID, datasetName
469
470        HDF5CloseFile fileID
471       
472        return result
473End     // The attribute waves are automatically killed since they are free waves
474
475
476
477//     
478//Function H_Test_HDFWriteTrans(fname,val)
479//      String fname
480//      Variable val
481//     
482//     
483//      String str
484//      PathInfo home
485//      str = S_path
486//     
487//      H_WriteTransmissionToHeader(str+fname,val)
488//     
489//      return(0)
490//End
491//
492//Function H_WriteTransmissionToHeader(fname,trans)
493//      String fname
494//      Variable trans
495//     
496//      Make/O/D/N=1 wTmpWrite
497//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
498//      String varName = "TRNS"
499//      wTmpWrite[0] = trans //
500//
501//      variable err
502//      err = HDFWrite_Wave(fname, groupName, varName, wTmpWrite)
503//      KillWaves wTmpWrite
504//     
505//      //err not handled here
506//             
507//      return(0)
508//End
509
510
511
512Function H_Test_ListAttributes(fname,groupName)
513        String fname,groupName
514        Variable trans
515       
516//      Make/O/D/N=1 wTmpWrite
517//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
518//      String varName = "TRNS"
519//      wTmpWrite[0] = trans //
520        String str
521        PathInfo home
522        str = S_path
523       
524        variable err
525        err = H_HDF_ListAttributes(str+fname, groupName)
526       
527        //err not handled here
528               
529        return(0)
530End
531
532Function H_HDF_ListAttributes(fname, groupName)
533        String fname, groupName
534       
535        variable err=0, fileID,groupID
536        String cDF = getDataFolder(1), temp
537        String NXentry_name, attrValue=""
538       
539        STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
540        InitHDF5DataInfo(di)    // Initialize structure.
541       
542        try     
543                HDF5OpenFile /Z fileID  as fname  //open file read-write
544                if(!fileID)
545                        err = 1
546                        abort "HDF5 file does not exist"
547                endif
548               
549                HDF5OpenGroup /Z fileID , groupName, groupID
550
551        //      (QUOKKA) !! At the moment, there is no entry for sample thickness in our data file
552        //      therefore create new HDF5 group to enable write / patch command
553        //      comment out the following group creation once thickness appears in revised file
554       
555                if(!groupID)
556                        HDF5CreateGroup /Z fileID, groupName, groupID
557                        //err = 1
558                        //abort "HDF5 group does not exist"
559                else
560
561//                      HDF5AttributeInfo(fileID, "/", 1, "file_name", 0, di)
562                        HDF5AttributeInfo(fileID, "/", 1, "NeXus_version", 0, di)
563                        Print di
564
565//                      see the HDF5 Browser  for how to get the actual <value> of the attribute. See GetPreviewString in
566//        or in FillGroupAttributesList or in FillDatasetAttributesList (from FillLists)
567//                      it seems to be ridiculously complex to get such a simple bit of information - the HDF5BrowserData STRUCT
568//                      needs to be filled first. Ugh.
569                        attrValue = GetPreviewString(fileID, 1, di, "/entry", "cucumber")
570                        Print "attrValue = ",attrValue
571                       
572                       
573                        //get attributes and save them
574                        HDF5ListAttributes/TYPE=1 /Z fileID, groupName          //TYPE=1 means that we're referencing a group, not a dataset
575                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
576                       
577                        // passing the groupID works too, then the group name is not needed                     
578                        HDF5ListAttributes/TYPE=1 /Z groupID, "."               //TYPE=1 means that we're referencing a group, not a dataset
579                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
580                endif
581        catch
582
583                // catch any aborts here
584               
585        endtry
586       
587        if(groupID)
588                HDF5CloseGroup /Z groupID
589        endif
590       
591        if(fileID)
592                HDF5CloseFile /Z fileID
593        endif
594
595        setDataFolder $cDF
596        return err
597end
598
599
600/////////////// end of the testing procedures ////////////////
Note: See TracBrowser for help on using the repository browser.