source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/VC_HDF5_VSANS_Utils.ipf @ 1023

Last change on this file since 1023 was 1023, checked in by srkline, 6 years ago

many additions.

Moved unused igor/nexus testing files to Vx_ prefix since they're garbage. Pulled out the useful bits for mask and div R/W and moved those to theire appropriate procedures.

Testing the simple correction of data, only tested basic subtraction. All of it still needs to be verified since I don't have any real header numbers and units yet.

Adjusted the columns on the file catalog to be more appropriate, and added a hook to allow loading of raw data files directly from the table and a popup contextual menu. May add more functionality to it later.

Corrected how the 1D data is plotted so that it correctly uses the binning type. I(q) save now also uses the binning as specified.

File size: 15.2 KB
Line 
1#pragma rtGlobals=1             // Use modern global access method.
2
3//
4// ********
5// TODO -- figure out how much of this file is really used, and how much is
6//    simply garbage now. I make "fake" data files by starting with a real data file
7//    written from NICE, and then I re-write proper values and data arrays to it from
8//    simulations in VCALC. So I do not need to define the structure myself.
9//
10//   JAN 2017
11//
12//
13//
14//
15//
16
17//
18// This file has utility procedures to be able to read/write
19// test HDF5 (from Igor) for SANS and VSANS in Nexus format
20//
21// It is predominantly to be able to WRITE a full Nexus file from Igor
22// -- this is only necessary to be able to write out "fake" VSANS files from VCALC
23//    since I need a "template" of the Nexus folder structure to start from and fill in
24//
25// It doesn't reproduce the NICE logs, but will leave a space for them if
26// it is read in and is part of the xref.
27//
28// It may be easier to hard-wire my own folder definition (NewDataFolder)
29// that is MY Nexus file to write out and use that as the base case
30// rather than rely on Pete's code, which is terribly slow for writing.
31// Attributes are the issue, but it may be unimportant if they are missing.
32//
33
34
35
36//
37// There's an odd sequence of steps that need to be done to use HDFGateway
38// to most easily write out attributes, and be able to read then back in.
39// Rather awkward, but it works without me needing to go nuts with an atomistic
40// understanding of HDF5. This is necessary for me to be able to write out
41// Nexus files, which is important both for testing, and for simulation.
42//
43// This is done as a set of macros that need to be applied in a specific sequence
44// in order to be able to generate HDF5___xref
45// Ideally, the HDF5___xref wave can be saved/imported to make proper saving possible
46// once the "tree" is finalized.
47//
48//
49// The basic saving routines are here, just as Proc rather than Macro menu clutter
50//
51
52//
53
54//
55// FEB 2017: All that is functional right now is the Setup/save of DIV and MASK files, and the attribute routines.
56//
57// The DIV and MASK routines are in their respective procedure files, and call save functions for the simplified
58// Nexus structure (in V_HDF5_RW_Utils.ipf).
59//
60// The R/W with attributes is part of the HDF5 Gateway procedure from Pete Jemain, and needs to be kept and functional
61// to be able to work with attributes - if needed.
62//
63Menu "VSANS"
64        SubMenu "Nexus File RW"
65//              "Fill_Nexus_V_Template"
66//              "Save_Nexus_V_Template"
67//              "Load_Nexus_V_Template"
68//              "-"
69//              "IgorOnly_Setup_VSANS_Struct"
70//              "IgorOnly_Save_VSANS_Nexus"
71//              "IgorOnly_Setup_SANS_Struct"
72//              "IgorOnly_Save_SANS_Struct"
73                "Setup_VSANS_DIV_Struct"
74                "Save_VSANS_DIV_Nexus"
75                "Setup_VSANS_MASK_Struct"
76                "Save_VSANS_MASK_Nexus"
77                "-"
78                "Read_Nexus with attributes",Read_Nexus_Xref()          //this will read with attributes
79                "Write_Nexus with attributes",Write_Nexus_Xref()                                //this will write out with attributes if read in by Read_Nexus_Xref
80        End
81End
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99/////////////////////////////
100
101
102
103Proc Load_Nexus_V_Template()
104        H_HDF5Gate_Read_Raw("")
105        String tmpStr=root:file_name  //SRK - so I can get the file name that was loaded
106       
107// this is the folder name
108        String base_name = StringFromList(0,File_Name,".")
109        RenameDataFolder $("root:"+base_name), V_Nexus_Template
110End
111
112
113Proc Fill_Nexus_V_Template()
114        H_Fill_VSANS_Template_wSim()
115End
116
117
118Proc Save_Nexus_V_Template()
119        H_HDF5Gate_Write_Raw("root:V_Nexus_Template:","")
120End
121
122
123
124/////////////below is largely depricated, ugly dance to be able to "fake" a file from Igor
125// which was not complete anyways.
126
127
128
129
130Proc IgorOnly_Setup_VSANS_Struct()
131
132        // lays out the tree and fills with dummy values
133        H_Setup_VSANS_Structure()
134       
135        // writes in the attributes
136        H_Fill_VSANS_Attributes()
137       
138        // fill in with VCALC simulation bits
139        H_Fill_VSANS_wSim()
140       
141End
142
143Proc IgorOnly_Save_VSANS_Nexus(fileName)
144        String fileName="Test_VSANS_file"
145
146        // save as HDF5 (no attributes saved yet)
147        Save_VSANS_file("root:VSANS_file", fileName+".h5")
148       
149        // read in a data file using the gateway-- reads from the home path
150        H_HDF5Gate_Read_Raw(fileName+".h5")
151       
152        // after reading in a "partial" file using the gateway (to generate the xref)
153        // Save the xref to disk (for later use)
154        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
155       
156        // after you've generated the HDF5___xref, load it in and copy it
157        // to the necessary folder location.
158        Copy_HDF5___xref("root:VSANS_file", "HDF5___xref")
159       
160        // writes out the contents of a data folder using the gateway
161        H_HDF5Gate_Write_Raw("root:VSANS_file", fileName+".h5")
162
163        // re-load the data file using the gateway-- reads from the home path
164        // now with attributes
165        H_HDF5Gate_Read_Raw(fileName+".h5")
166       
167End
168
169
170
171Proc IgorOnly_Setup_SANS_Struct()
172
173        // lays out the tree and fills with dummy values
174        H_Setup_SANS_Structure()
175       
176        // writes in the attributes
177        H_Fill_SANS_Attributes()
178       
179        // fill in with VCALC simulation bits
180        H_Fill_SANS_wSim()
181
182End
183
184Proc IgorOnly_Save_SANS_Nexus(fileName)
185        String fileName="Test_SANS_file"
186       
187        // save as HDF5 (no attributes saved yet) (save_VSANS is actually generic HDF...)
188        Save_VSANS_file("root:SANS_file", fileName+".h5")
189       
190        // read in a data file using the gateway-- reads from the home path
191        H_HDF5Gate_Read_Raw(fileName+".h5")
192       
193        // after reading in a "partial" file using the gateway (to generate the xref)
194        // Save the xref to disk (for later use)
195        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
196       
197        // after you've generated the HDF5___xref, load it in and copy it
198        // to the necessary folder location.
199        Copy_HDF5___xref("root:SANS_file", "HDF5___xref")
200       
201        // writes out the contents of a data folder using the gateway
202        H_HDF5Gate_Write_Raw("root:SANS_file", fileName+".h5")
203
204        // re-load the data file using the gateway-- reads from the home path
205        // now with attributes
206        H_HDF5Gate_Read_Raw(fileName+".h5")
207
208
209End
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229// passing null file string presents a dialog
230// these two procedures will use the full Xrefs, so they can write out full
231// files. They are, however, slow
232Proc Read_Nexus_Xref()
233        H_HDF5Gate_Read_Raw("")
234End
235
236Proc Write_Nexus_Xref()
237        H_HDF5Gate_Write_Raw()
238End
239
240//
241// writes out the contents of a data folder using the gateway
242// -- the HDF5___xref wave must be present at the top level for it to
243//    write out anything.
244//
245Proc H_HDF5Gate_Write_Raw(dfPath,filename)
246        String dfPath   = "root:"       // e.g., "root:FolderA" or ":"
247        String filename = "Test_VSANS_file.h5"
248//      Prompt dfPath, "Data Folder",popup,H_GetAList(4)       
249//      Prompt fileName, "File name"
250       
251//      DoPrompt "Folder and file to write", dfPath,fileName
252        // Check our work so far.
253        // If something prints, there was an error above.
254//      print H5GW_ValidateFolder(dfPath)
255
256// H5GW_WriteHDF5 calls the validate function, so no need to call it before
257        print H5GW_WriteHDF5(dfPath, filename)
258       
259        SetDataFolder root:
260End
261
262//
263// read in a data file using the gateway
264//
265// reads from the home path
266//
267Proc H_HDF5Gate_Read_Raw(file)
268        String file
269       
270//      NewDataFolder/O/S root:newdata
271        Print H5GW_ReadHDF5("", file)   // reads into current folder
272        SetDataFolder root:
273End
274
275
276
277// this is in PlotUtils, but duplicated here
278// utility used in the "PlotSmeared...() macros to get a list of data folders
279//
280//1:    Waves.
281//2:    Numeric variables.
282//3:    String variables.
283//4:    Data folders.
284Function/S H_GetAList(type)
285        Variable type
286       
287        SetDataFolder root:
288       
289        String objName,str=""
290        Variable index = 0
291        do
292                objName = GetIndexedObjName(":", type, index)
293                if (strlen(objName) == 0)
294                        break
295                endif
296                //Print objName
297                str += objName + ";"
298                index += 1
299        while(1)
300       
301        // remove myGlobals, Packages, etc. from the folder list
302        if(type==4)
303                str = RemoveFromList("myGlobals", str , ";" )
304                str = RemoveFromList("Packages", str, ";")
305        endif
306       
307        return(str)
308End
309
310//
311// after reading in a "partial" file using the gateway (to generate the xref)
312// Save the xref to disk (for later use)
313//
314Proc Save_HDF5___xref(dfPath, filename)
315        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
316        String filename = "HDF5___xref"
317
318        Save/T/P=home $(dfPath+":HDF5___xref") as "HDF5___xref.itx"
319
320//      Copy_HDF5___xref(dfPath, filename)
321       
322        SetDataFolder root:
323End
324
325//
326// after you've generated the HDF5___xref, load it in and copy it to
327// the necessary folder location.
328// - then all is set to *really* write out the file correctly, including the attributes
329//
330Proc Copy_HDF5___xref(dfPath, filename)
331        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
332        String filename = "HDF5___xref"
333
334        if(exists(filename) != 1)
335                //load it in   
336                LoadWave/T/P=home/O "HDF5___xref.itx"
337        endif
338       
339        Duplicate/O HDF5___xref, $(dfPath+":HDF5___xref")
340       
341        SetDataFolder root:
342End
343
344
345
346//////// testing procedures, may be of use, maybe not //////////////
347
348
349//////// Two procedures that test out Pete Jemain's HDF5Gateway
350//
351// This works fine, but it may not be terribly compatible with the way NICE will eventually
352// write out the data files. I'll have very little control over that and I'll need to cobble together
353// a bunch of fixes to cover up their mistakes.
354//
355// Using Nick Hauser's code as a starting point may be a lot more painful, but more flexible in the end.
356//
357// I'm completely baffled about what to do with attributes. Are they needed, is this the best way to deal
358// with them, do I care about reading them in, and if I do, why?
359//
360Proc H_HDF5Gate_WriteTest()
361
362        // create the folder structure
363        NewDataFolder/O/S root:mydata
364        NewDataFolder/O sasentry
365        NewDataFolder/O :sasentry:sasdata
366
367        // create the waves
368        Make/O :sasentry:sasdata:I0
369        Make/O :sasentry:sasdata:Q0
370
371        Make/O/N=0 Igor___folder_attributes
372        Make/O/N=0 :sasentry:Igor___folder_attributes
373        Make/O/N=0 :sasentry:sasdata:Igor___folder_attributes
374
375        // create the attributes
376        Note/K Igor___folder_attributes, "producer=IgorPro\rNX_class=NXroot"
377        Note/K :sasentry:Igor___folder_attributes, "NX_class=NXentry"
378        Note/K :sasentry:sasdata:Igor___folder_attributes, "NX_class=NXdata"
379        Note/K :sasentry:sasdata:I0, "units=1/cm\rsignal=1\rtitle=reduced intensity"
380        Note/K :sasentry:sasdata:Q0, "units=1/A\rtitle=|scattering vector|"
381
382        // create the cross-reference mapping
383        Make/O/T/N=(5,2) HDF5___xref
384        Edit/K=0 'HDF5___xref';DelayUpdate
385        HDF5___xref[0][1] = ":"
386        HDF5___xref[1][1] = ":sasentry"
387        HDF5___xref[2][1] = ":sasentry:sasdata"
388        HDF5___xref[3][1] = ":sasentry:sasdata:I0"
389        HDF5___xref[4][1] = ":sasentry:sasdata:Q0"
390        HDF5___xref[0][0] = "/"
391        HDF5___xref[1][0] = "/sasentry"
392        HDF5___xref[2][0] = "/sasentry/sasdata"
393        HDF5___xref[3][0] = "/sasentry/sasdata/I"
394        HDF5___xref[4][0] = "/sasentry/sasdata/Q"
395
396        // Check our work so far.
397        // If something prints, there was an error above.
398        print H5GW_ValidateFolder("root:mydata")
399
400        // set I0 and Q0 to your data
401
402        print H5GW_WriteHDF5("root:mydata", "mydata.h5")
403       
404        SetDataFolder root:
405End
406
407
408//
409// given a filename of a VSANS data file of the form
410// name.anything.any.other (any number of extensions is OK)
411// returns the name as a string without ANY ".fbdfasga" extension
412//
413// returns the input string if a "." can't be found (maybe it wasn't there)
414//
415Function/S H_RemoveDotExtension(item)
416        String item
417        String invalid = item   //
418        Variable num=-1
419       
420        //find the "dot"
421        String runStr=""
422        Variable pos = strsearch(item,".",0)
423        if(pos == -1)
424                //"dot" not found
425                return (invalid)
426        else
427                //found, get all of the characters preceeding it
428                runStr = item[0,pos-1]
429                return (runStr)
430        Endif
431End
432
433
434//
435// Writing attributes to a group and to a dataset.
436// example from the WM documentation for HDF5SaveData
437//
438Function DemoAttributes(w)
439        Wave w
440
441        Variable result = 0     // 0 means no error
442       
443        // Create file
444        Variable fileID
445        HDF5CreateFile/P=home /O /Z fileID as "Test.h5"
446        if (V_flag != 0)
447                Print "HDF5CreateFile failed"
448                return -1
449        endif
450
451        // Write an attribute to the root group
452        Make /FREE /T /N=1 groupAttribute = "This is a group attribute"
453        HDF5SaveData /A="GroupAttribute" groupAttribute, fileID, "/"
454       
455        // Save wave as dataset
456        HDF5SaveData /O /Z w, fileID    // Uses wave name as dataset name
457        if (V_flag != 0)
458                Print "HDF5SaveData failed"
459                result = -1
460        endif
461
462        // Write an attribute to the dataset
463        Make /FREE /T /N=1 datasetAttribute = "This is a dataset attribute"
464        String datasetName = NameOfWave(w)
465        HDF5SaveData /A="DatasetAttribute" datasetAttribute, fileID, datasetName
466
467        HDF5CloseFile fileID
468       
469        return result
470End     // The attribute waves are automatically killed since they are free waves
471
472
473
474//     
475//Function H_Test_HDFWriteTrans(fname,val)
476//      String fname
477//      Variable val
478//     
479//     
480//      String str
481//      PathInfo home
482//      str = S_path
483//     
484//      H_WriteTransmissionToHeader(str+fname,val)
485//     
486//      return(0)
487//End
488//
489//Function H_WriteTransmissionToHeader(fname,trans)
490//      String fname
491//      Variable trans
492//     
493//      Make/O/D/N=1 wTmpWrite
494//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
495//      String varName = "TRNS"
496//      wTmpWrite[0] = trans //
497//
498//      variable err
499//      err = HDFWrite_Wave(fname, groupName, varName, wTmpWrite)
500//      KillWaves wTmpWrite
501//     
502//      //err not handled here
503//             
504//      return(0)
505//End
506
507
508
509Function H_Test_ListAttributes(fname,groupName)
510        String fname,groupName
511        Variable trans
512       
513//      Make/O/D/N=1 wTmpWrite
514//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
515//      String varName = "TRNS"
516//      wTmpWrite[0] = trans //
517        String str
518        PathInfo home
519        str = S_path
520       
521        variable err
522        err = H_HDF_ListAttributes(str+fname, groupName)
523       
524        //err not handled here
525               
526        return(0)
527End
528
529Function H_HDF_ListAttributes(fname, groupName)
530        String fname, groupName
531       
532        variable err=0, fileID,groupID
533        String cDF = getDataFolder(1), temp
534        String NXentry_name, attrValue=""
535       
536        STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
537        InitHDF5DataInfo(di)    // Initialize structure.
538       
539        try     
540                HDF5OpenFile /Z fileID  as fname  //open file read-write
541                if(!fileID)
542                        err = 1
543                        abort "HDF5 file does not exist"
544                endif
545               
546                HDF5OpenGroup /Z fileID , groupName, groupID
547
548        //      (QUOKKA) !! At the moment, there is no entry for sample thickness in our data file
549        //      therefore create new HDF5 group to enable write / patch command
550        //      comment out the following group creation once thickness appears in revised file
551       
552                if(!groupID)
553                        HDF5CreateGroup /Z fileID, groupName, groupID
554                        //err = 1
555                        //abort "HDF5 group does not exist"
556                else
557
558//                      HDF5AttributeInfo(fileID, "/", 1, "file_name", 0, di)
559                        HDF5AttributeInfo(fileID, "/", 1, "NeXus_version", 0, di)
560                        Print di
561
562//                      see the HDF5 Browser  for how to get the actual <value> of the attribute. See GetPreviewString in
563//        or in FillGroupAttributesList or in FillDatasetAttributesList (from FillLists)
564//                      it seems to be ridiculously complex to get such a simple bit of information - the HDF5BrowserData STRUCT
565//                      needs to be filled first. Ugh.
566                        attrValue = GetPreviewString(fileID, 1, di, "/entry", "cucumber")
567                        Print "attrValue = ",attrValue
568                       
569                       
570                        //get attributes and save them
571                        HDF5ListAttributes/TYPE=1 /Z fileID, groupName          //TYPE=1 means that we're referencing a group, not a dataset
572                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
573                       
574                        // passing the groupID works too, then the group name is not needed                     
575                        HDF5ListAttributes/TYPE=1 /Z groupID, "."               //TYPE=1 means that we're referencing a group, not a dataset
576                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
577                endif
578        catch
579
580                // catch any aborts here
581               
582        endtry
583       
584        if(groupID)
585                HDF5CloseGroup /Z groupID
586        endif
587       
588        if(fileID)
589                HDF5CloseFile /Z fileID
590        endif
591
592        setDataFolder $cDF
593        return err
594end
595
596
597/////////////// end of the testing procedures ////////////////
Note: See TracBrowser for help on using the repository browser.