source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/VC_HDF5_VSANS_Utils.ipf @ 1129

Last change on this file since 1129 was 1042, checked in by srkline, 5 years ago

moved dead time correction to before the solid angle correction, so that the dead time would be correcting counts, not counts per solid angle

added a routine to kill all of the waves and folders possible, if the overall DF kill failed. This is to prevent stray folders and waves from being present if different data files are loaded - since different data blocks are present for say, 3He data, data with temperature logging, etc.
This kill routine is used every time, before raw data is loaded, DIV or MASK loaded, or data is converted to WORK.

changed the "Save I(q)" button on the data display panel to save as ITX format, since the data has not been processed, and data can more easily be used for trimming input.

picking protocols in dialogs now excludes/includes appropriate waves

menus are consolidated

Fixed bug in SANS macros where the DRK[] item in the protocol could be null, and force the read of a DRK file, even if it was not desired.

File size: 14.6 KB
Line 
1#pragma rtGlobals=1             // Use modern global access method.
2
3//
4// ********
5// TODO -- figure out how much of this file is really used, and how much is
6//    simply garbage now. I make "fake" data files by starting with a real data file
7//    written from NICE, and then I re-write proper values and data arrays to it from
8//    simulations in VCALC. So I do not need to define the structure myself.
9//
10//   JAN 2017
11//
12//
13//
14//
15//
16
17//
18// This file has utility procedures to be able to read/write
19// test HDF5 (from Igor) for SANS and VSANS in Nexus format
20//
21// It is predominantly to be able to WRITE a full Nexus file from Igor
22// -- this is only necessary to be able to write out "fake" VSANS files from VCALC
23//    since I need a "template" of the Nexus folder structure to start from and fill in
24//
25// It doesn't reproduce the NICE logs, but will leave a space for them if
26// it is read in and is part of the xref.
27//
28// It may be easier to hard-wire my own folder definition (NewDataFolder)
29// that is MY Nexus file to write out and use that as the base case
30// rather than rely on Pete's code, which is terribly slow for writing.
31// Attributes are the issue, but it may be unimportant if they are missing.
32//
33
34
35
36//
37// There's an odd sequence of steps that need to be done to use HDFGateway
38// to most easily write out attributes, and be able to read then back in.
39// Rather awkward, but it works without me needing to go nuts with an atomistic
40// understanding of HDF5. This is necessary for me to be able to write out
41// Nexus files, which is important both for testing, and for simulation.
42//
43// This is done as a set of macros that need to be applied in a specific sequence
44// in order to be able to generate HDF5___xref
45// Ideally, the HDF5___xref wave can be saved/imported to make proper saving possible
46// once the "tree" is finalized.
47//
48//
49// The basic saving routines are here, just as Proc rather than Macro menu clutter
50//
51
52//
53
54//
55// FEB 2017: All that is functional right now is the Setup/save of DIV and MASK files, and the attribute routines.
56//
57// The DIV and MASK routines are in their respective procedure files, and call save functions for the simplified
58// Nexus structure (in V_HDF5_RW_Utils.ipf).
59//
60// The R/W with attributes is part of the HDF5 Gateway procedure from Pete Jemain, and needs to be kept and functional
61// to be able to work with attributes - if needed.
62//
63
64
65
66
67
68/////////////////////////////
69
70
71
72Proc Load_Nexus_V_Template()
73        H_HDF5Gate_Read_Raw("")
74        String tmpStr=root:file_name  //SRK - so I can get the file name that was loaded
75       
76// this is the folder name
77        String base_name = StringFromList(0,File_Name,".")
78        RenameDataFolder $("root:"+base_name), V_Nexus_Template
79End
80
81
82Proc Fill_Nexus_V_Template()
83        H_Fill_VSANS_Template_wSim()
84End
85
86
87Proc Save_Nexus_V_Template()
88        H_HDF5Gate_Write_Raw("root:V_Nexus_Template:","")
89End
90
91
92
93/////////////below is largely depricated, ugly dance to be able to "fake" a file from Igor
94// which was not complete anyways.
95
96
97
98
99Proc IgorOnly_Setup_VSANS_Struct()
100
101        // lays out the tree and fills with dummy values
102        H_Setup_VSANS_Structure()
103       
104        // writes in the attributes
105        H_Fill_VSANS_Attributes()
106       
107        // fill in with VCALC simulation bits
108        H_Fill_VSANS_wSim()
109       
110End
111
112Proc IgorOnly_Save_VSANS_Nexus(fileName)
113        String fileName="Test_VSANS_file"
114
115        // save as HDF5 (no attributes saved yet)
116        Save_VSANS_file("root:VSANS_file", fileName+".h5")
117       
118        // read in a data file using the gateway-- reads from the home path
119        H_HDF5Gate_Read_Raw(fileName+".h5")
120       
121        // after reading in a "partial" file using the gateway (to generate the xref)
122        // Save the xref to disk (for later use)
123        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
124       
125        // after you've generated the HDF5___xref, load it in and copy it
126        // to the necessary folder location.
127        Copy_HDF5___xref("root:VSANS_file", "HDF5___xref")
128       
129        // writes out the contents of a data folder using the gateway
130        H_HDF5Gate_Write_Raw("root:VSANS_file", fileName+".h5")
131
132        // re-load the data file using the gateway-- reads from the home path
133        // now with attributes
134        H_HDF5Gate_Read_Raw(fileName+".h5")
135       
136End
137
138
139
140Proc IgorOnly_Setup_SANS_Struct()
141
142        // lays out the tree and fills with dummy values
143        H_Setup_SANS_Structure()
144       
145        // writes in the attributes
146        H_Fill_SANS_Attributes()
147       
148        // fill in with VCALC simulation bits
149        H_Fill_SANS_wSim()
150
151End
152
153Proc IgorOnly_Save_SANS_Nexus(fileName)
154        String fileName="Test_SANS_file"
155       
156        // save as HDF5 (no attributes saved yet) (save_VSANS is actually generic HDF...)
157        Save_VSANS_file("root:SANS_file", fileName+".h5")
158       
159        // read in a data file using the gateway-- reads from the home path
160        H_HDF5Gate_Read_Raw(fileName+".h5")
161       
162        // after reading in a "partial" file using the gateway (to generate the xref)
163        // Save the xref to disk (for later use)
164        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
165       
166        // after you've generated the HDF5___xref, load it in and copy it
167        // to the necessary folder location.
168        Copy_HDF5___xref("root:SANS_file", "HDF5___xref")
169       
170        // writes out the contents of a data folder using the gateway
171        H_HDF5Gate_Write_Raw("root:SANS_file", fileName+".h5")
172
173        // re-load the data file using the gateway-- reads from the home path
174        // now with attributes
175        H_HDF5Gate_Read_Raw(fileName+".h5")
176
177
178End
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198// passing null file string presents a dialog
199// these two procedures will use the full Xrefs, so they can write out full
200// files. They are, however, slow
201Proc Read_Nexus_Xref()
202        H_HDF5Gate_Read_Raw("")
203End
204
205Proc Write_Nexus_Xref()
206        H_HDF5Gate_Write_Raw()
207End
208
209//
210// writes out the contents of a data folder using the gateway
211// -- the HDF5___xref wave must be present at the top level for it to
212//    write out anything.
213//
214Proc H_HDF5Gate_Write_Raw(dfPath,filename)
215        String dfPath   = "root:"       // e.g., "root:FolderA" or ":"
216        String filename = "Test_VSANS_file.h5"
217//      Prompt dfPath, "Data Folder",popup,H_GetAList(4)       
218//      Prompt fileName, "File name"
219       
220//      DoPrompt "Folder and file to write", dfPath,fileName
221        // Check our work so far.
222        // If something prints, there was an error above.
223//      print H5GW_ValidateFolder(dfPath)
224
225// H5GW_WriteHDF5 calls the validate function, so no need to call it before
226        print H5GW_WriteHDF5(dfPath, filename)
227       
228        SetDataFolder root:
229End
230
231//
232// read in a data file using the gateway
233//
234// reads from the home path
235//
236Proc H_HDF5Gate_Read_Raw(file)
237        String file
238       
239//      NewDataFolder/O/S root:newdata
240        Print H5GW_ReadHDF5("", file)   // reads into current folder
241        SetDataFolder root:
242End
243
244
245
246// this is in PlotUtils, but duplicated here
247// utility used in the "PlotSmeared...() macros to get a list of data folders
248//
249//1:    Waves.
250//2:    Numeric variables.
251//3:    String variables.
252//4:    Data folders.
253Function/S H_GetAList(type)
254        Variable type
255       
256        SetDataFolder root:
257       
258        String objName,str=""
259        Variable index = 0
260        do
261                objName = GetIndexedObjName(":", type, index)
262                if (strlen(objName) == 0)
263                        break
264                endif
265                //Print objName
266                str += objName + ";"
267                index += 1
268        while(1)
269       
270        // remove myGlobals, Packages, etc. from the folder list
271        if(type==4)
272                str = RemoveFromList("myGlobals", str , ";" )
273                str = RemoveFromList("Packages", str, ";")
274        endif
275       
276        return(str)
277End
278
279//
280// after reading in a "partial" file using the gateway (to generate the xref)
281// Save the xref to disk (for later use)
282//
283Proc Save_HDF5___xref(dfPath, filename)
284        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
285        String filename = "HDF5___xref"
286
287        Save/T/P=home $(dfPath+":HDF5___xref") as "HDF5___xref.itx"
288
289//      Copy_HDF5___xref(dfPath, filename)
290       
291        SetDataFolder root:
292End
293
294//
295// after you've generated the HDF5___xref, load it in and copy it to
296// the necessary folder location.
297// - then all is set to *really* write out the file correctly, including the attributes
298//
299Proc Copy_HDF5___xref(dfPath, filename)
300        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
301        String filename = "HDF5___xref"
302
303        if(exists(filename) != 1)
304                //load it in   
305                LoadWave/T/P=home/O "HDF5___xref.itx"
306        endif
307       
308        Duplicate/O HDF5___xref, $(dfPath+":HDF5___xref")
309       
310        SetDataFolder root:
311End
312
313
314
315//////// testing procedures, may be of use, maybe not //////////////
316
317
318//////// Two procedures that test out Pete Jemain's HDF5Gateway
319//
320// This works fine, but it may not be terribly compatible with the way NICE will eventually
321// write out the data files. I'll have very little control over that and I'll need to cobble together
322// a bunch of fixes to cover up their mistakes.
323//
324// Using Nick Hauser's code as a starting point may be a lot more painful, but more flexible in the end.
325//
326// I'm completely baffled about what to do with attributes. Are they needed, is this the best way to deal
327// with them, do I care about reading them in, and if I do, why?
328//
329Proc H_HDF5Gate_WriteTest()
330
331        // create the folder structure
332        NewDataFolder/O/S root:mydata
333        NewDataFolder/O sasentry
334        NewDataFolder/O :sasentry:sasdata
335
336        // create the waves
337        Make/O :sasentry:sasdata:I0
338        Make/O :sasentry:sasdata:Q0
339
340        Make/O/N=0 Igor___folder_attributes
341        Make/O/N=0 :sasentry:Igor___folder_attributes
342        Make/O/N=0 :sasentry:sasdata:Igor___folder_attributes
343
344        // create the attributes
345        Note/K Igor___folder_attributes, "producer=IgorPro\rNX_class=NXroot"
346        Note/K :sasentry:Igor___folder_attributes, "NX_class=NXentry"
347        Note/K :sasentry:sasdata:Igor___folder_attributes, "NX_class=NXdata"
348        Note/K :sasentry:sasdata:I0, "units=1/cm\rsignal=1\rtitle=reduced intensity"
349        Note/K :sasentry:sasdata:Q0, "units=1/A\rtitle=|scattering vector|"
350
351        // create the cross-reference mapping
352        Make/O/T/N=(5,2) HDF5___xref
353        Edit/K=0 'HDF5___xref';DelayUpdate
354        HDF5___xref[0][1] = ":"
355        HDF5___xref[1][1] = ":sasentry"
356        HDF5___xref[2][1] = ":sasentry:sasdata"
357        HDF5___xref[3][1] = ":sasentry:sasdata:I0"
358        HDF5___xref[4][1] = ":sasentry:sasdata:Q0"
359        HDF5___xref[0][0] = "/"
360        HDF5___xref[1][0] = "/sasentry"
361        HDF5___xref[2][0] = "/sasentry/sasdata"
362        HDF5___xref[3][0] = "/sasentry/sasdata/I"
363        HDF5___xref[4][0] = "/sasentry/sasdata/Q"
364
365        // Check our work so far.
366        // If something prints, there was an error above.
367        print H5GW_ValidateFolder("root:mydata")
368
369        // set I0 and Q0 to your data
370
371        print H5GW_WriteHDF5("root:mydata", "mydata.h5")
372       
373        SetDataFolder root:
374End
375
376
377//
378// given a filename of a VSANS data file of the form
379// name.anything.any.other (any number of extensions is OK)
380// returns the name as a string without ANY ".fbdfasga" extension
381//
382// returns the input string if a "." can't be found (maybe it wasn't there)
383//
384Function/S H_RemoveDotExtension(item)
385        String item
386        String invalid = item   //
387        Variable num=-1
388       
389        //find the "dot"
390        String runStr=""
391        Variable pos = strsearch(item,".",0)
392        if(pos == -1)
393                //"dot" not found
394                return (invalid)
395        else
396                //found, get all of the characters preceeding it
397                runStr = item[0,pos-1]
398                return (runStr)
399        Endif
400End
401
402
403//
404// Writing attributes to a group and to a dataset.
405// example from the WM documentation for HDF5SaveData
406//
407Function DemoAttributes(w)
408        Wave w
409
410        Variable result = 0     // 0 means no error
411       
412        // Create file
413        Variable fileID
414        HDF5CreateFile/P=home /O /Z fileID as "Test.h5"
415        if (V_flag != 0)
416                Print "HDF5CreateFile failed"
417                return -1
418        endif
419
420        // Write an attribute to the root group
421        Make /FREE /T /N=1 groupAttribute = "This is a group attribute"
422        HDF5SaveData /A="GroupAttribute" groupAttribute, fileID, "/"
423       
424        // Save wave as dataset
425        HDF5SaveData /O /Z w, fileID    // Uses wave name as dataset name
426        if (V_flag != 0)
427                Print "HDF5SaveData failed"
428                result = -1
429        endif
430
431        // Write an attribute to the dataset
432        Make /FREE /T /N=1 datasetAttribute = "This is a dataset attribute"
433        String datasetName = NameOfWave(w)
434        HDF5SaveData /A="DatasetAttribute" datasetAttribute, fileID, datasetName
435
436        HDF5CloseFile fileID
437       
438        return result
439End     // The attribute waves are automatically killed since they are free waves
440
441
442
443//     
444//Function H_Test_HDFWriteTrans(fname,val)
445//      String fname
446//      Variable val
447//     
448//     
449//      String str
450//      PathInfo home
451//      str = S_path
452//     
453//      H_WriteTransmissionToHeader(str+fname,val)
454//     
455//      return(0)
456//End
457//
458//Function H_WriteTransmissionToHeader(fname,trans)
459//      String fname
460//      Variable trans
461//     
462//      Make/O/D/N=1 wTmpWrite
463//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
464//      String varName = "TRNS"
465//      wTmpWrite[0] = trans //
466//
467//      variable err
468//      err = HDFWrite_Wave(fname, groupName, varName, wTmpWrite)
469//      KillWaves wTmpWrite
470//     
471//      //err not handled here
472//             
473//      return(0)
474//End
475
476
477
478Function H_Test_ListAttributes(fname,groupName)
479        String fname,groupName
480        Variable trans
481       
482//      Make/O/D/N=1 wTmpWrite
483//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
484//      String varName = "TRNS"
485//      wTmpWrite[0] = trans //
486        String str
487        PathInfo home
488        str = S_path
489       
490        variable err
491        err = H_HDF_ListAttributes(str+fname, groupName)
492       
493        //err not handled here
494               
495        return(0)
496End
497
498Function H_HDF_ListAttributes(fname, groupName)
499        String fname, groupName
500       
501        variable err=0, fileID,groupID
502        String cDF = getDataFolder(1), temp
503        String NXentry_name, attrValue=""
504       
505        STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
506        InitHDF5DataInfo(di)    // Initialize structure.
507       
508        try     
509                HDF5OpenFile /Z fileID  as fname  //open file read-write
510                if(!fileID)
511                        err = 1
512                        abort "HDF5 file does not exist"
513                endif
514               
515                HDF5OpenGroup /Z fileID , groupName, groupID
516
517        //      (QUOKKA) !! At the moment, there is no entry for sample thickness in our data file
518        //      therefore create new HDF5 group to enable write / patch command
519        //      comment out the following group creation once thickness appears in revised file
520       
521                if(!groupID)
522                        HDF5CreateGroup /Z fileID, groupName, groupID
523                        //err = 1
524                        //abort "HDF5 group does not exist"
525                else
526
527//                      HDF5AttributeInfo(fileID, "/", 1, "file_name", 0, di)
528                        HDF5AttributeInfo(fileID, "/", 1, "NeXus_version", 0, di)
529                        Print di
530
531//                      see the HDF5 Browser  for how to get the actual <value> of the attribute. See GetPreviewString in
532//        or in FillGroupAttributesList or in FillDatasetAttributesList (from FillLists)
533//                      it seems to be ridiculously complex to get such a simple bit of information - the HDF5BrowserData STRUCT
534//                      needs to be filled first. Ugh.
535                        attrValue = GetPreviewString(fileID, 1, di, "/entry", "cucumber")
536                        Print "attrValue = ",attrValue
537                       
538                       
539                        //get attributes and save them
540                        HDF5ListAttributes/TYPE=1 /Z fileID, groupName          //TYPE=1 means that we're referencing a group, not a dataset
541                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
542                       
543                        // passing the groupID works too, then the group name is not needed                     
544                        HDF5ListAttributes/TYPE=1 /Z groupID, "."               //TYPE=1 means that we're referencing a group, not a dataset
545                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
546                endif
547        catch
548
549                // catch any aborts here
550               
551        endtry
552       
553        if(groupID)
554                HDF5CloseGroup /Z groupID
555        endif
556       
557        if(fileID)
558                HDF5CloseFile /Z fileID
559        endif
560
561        setDataFolder $cDF
562        return err
563end
564
565
566/////////////// end of the testing procedures ////////////////
Note: See TracBrowser for help on using the repository browser.