source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/VC_HDF5_VSANS_Utils.ipf @ 1248

Last change on this file since 1248 was 1242, checked in by srkline, 3 years ago

updating the IgorVersion? pragma to v7.0 for all files to be consistent.

File size: 14.6 KB
Line 
1#pragma rtGlobals=1             // Use modern global access method.
2#pragma IgorVersion = 7.00
3
4//
5// ********
6// TODO -- figure out how much of this file is really used, and how much is
7//    simply garbage now. I make "fake" data files by starting with a real data file
8//    written from NICE, and then I re-write proper values and data arrays to it from
9//    simulations in VCALC. So I do not need to define the structure myself.
10//
11//   JAN 2017
12//
13//
14//
15//
16//
17
18//
19// This file has utility procedures to be able to read/write
20// test HDF5 (from Igor) for SANS and VSANS in Nexus format
21//
22// It is predominantly to be able to WRITE a full Nexus file from Igor
23// -- this is only necessary to be able to write out "fake" VSANS files from VCALC
24//    since I need a "template" of the Nexus folder structure to start from and fill in
25//
26// It doesn't reproduce the NICE logs, but will leave a space for them if
27// it is read in and is part of the xref.
28//
29// It may be easier to hard-wire my own folder definition (NewDataFolder)
30// that is MY Nexus file to write out and use that as the base case
31// rather than rely on Pete's code, which is terribly slow for writing.
32// Attributes are the issue, but it may be unimportant if they are missing.
33//
34
35
36
37//
38// There's an odd sequence of steps that need to be done to use HDFGateway
39// to most easily write out attributes, and be able to read then back in.
40// Rather awkward, but it works without me needing to go nuts with an atomistic
41// understanding of HDF5. This is necessary for me to be able to write out
42// Nexus files, which is important both for testing, and for simulation.
43//
44// This is done as a set of macros that need to be applied in a specific sequence
45// in order to be able to generate HDF5___xref
46// Ideally, the HDF5___xref wave can be saved/imported to make proper saving possible
47// once the "tree" is finalized.
48//
49//
50// The basic saving routines are here, just as Proc rather than Macro menu clutter
51//
52
53//
54
55//
56// FEB 2017: All that is functional right now is the Setup/save of DIV and MASK files, and the attribute routines.
57//
58// The DIV and MASK routines are in their respective procedure files, and call save functions for the simplified
59// Nexus structure (in V_HDF5_RW_Utils.ipf).
60//
61// The R/W with attributes is part of the HDF5 Gateway procedure from Pete Jemain, and needs to be kept and functional
62// to be able to work with attributes - if needed.
63//
64
65
66
67
68
69/////////////////////////////
70
71
72
73Proc Load_Nexus_V_Template()
74        H_HDF5Gate_Read_Raw("")
75        String tmpStr=root:file_name  //SRK - so I can get the file name that was loaded
76       
77// this is the folder name
78        String base_name = StringFromList(0,File_Name,".")
79        RenameDataFolder $("root:"+base_name), V_Nexus_Template
80End
81
82
83Proc Fill_Nexus_V_Template()
84        H_Fill_VSANS_Template_wSim()
85End
86
87
88Proc Save_Nexus_V_Template()
89        H_HDF5Gate_Write_Raw("root:V_Nexus_Template:","")
90End
91
92
93
94/////////////below is largely depricated, ugly dance to be able to "fake" a file from Igor
95// which was not complete anyways.
96
97
98
99
100Proc IgorOnly_Setup_VSANS_Struct()
101
102        // lays out the tree and fills with dummy values
103        H_Setup_VSANS_Structure()
104       
105        // writes in the attributes
106        H_Fill_VSANS_Attributes()
107       
108        // fill in with VCALC simulation bits
109        H_Fill_VSANS_wSim()
110       
111End
112
113Proc IgorOnly_Save_VSANS_Nexus(fileName)
114        String fileName="Test_VSANS_file"
115
116        // save as HDF5 (no attributes saved yet)
117        Save_VSANS_file("root:VSANS_file", fileName+".h5")
118       
119        // read in a data file using the gateway-- reads from the home path
120        H_HDF5Gate_Read_Raw(fileName+".h5")
121       
122        // after reading in a "partial" file using the gateway (to generate the xref)
123        // Save the xref to disk (for later use)
124        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
125       
126        // after you've generated the HDF5___xref, load it in and copy it
127        // to the necessary folder location.
128        Copy_HDF5___xref("root:VSANS_file", "HDF5___xref")
129       
130        // writes out the contents of a data folder using the gateway
131        H_HDF5Gate_Write_Raw("root:VSANS_file", fileName+".h5")
132
133        // re-load the data file using the gateway-- reads from the home path
134        // now with attributes
135        H_HDF5Gate_Read_Raw(fileName+".h5")
136       
137End
138
139
140
141Proc IgorOnly_Setup_SANS_Struct()
142
143        // lays out the tree and fills with dummy values
144        H_Setup_SANS_Structure()
145       
146        // writes in the attributes
147        H_Fill_SANS_Attributes()
148       
149        // fill in with VCALC simulation bits
150        H_Fill_SANS_wSim()
151
152End
153
154Proc IgorOnly_Save_SANS_Nexus(fileName)
155        String fileName="Test_SANS_file"
156       
157        // save as HDF5 (no attributes saved yet) (save_VSANS is actually generic HDF...)
158        Save_VSANS_file("root:SANS_file", fileName+".h5")
159       
160        // read in a data file using the gateway-- reads from the home path
161        H_HDF5Gate_Read_Raw(fileName+".h5")
162       
163        // after reading in a "partial" file using the gateway (to generate the xref)
164        // Save the xref to disk (for later use)
165        Save_HDF5___xref("root:"+fileName,"HDF5___xref")
166       
167        // after you've generated the HDF5___xref, load it in and copy it
168        // to the necessary folder location.
169        Copy_HDF5___xref("root:SANS_file", "HDF5___xref")
170       
171        // writes out the contents of a data folder using the gateway
172        H_HDF5Gate_Write_Raw("root:SANS_file", fileName+".h5")
173
174        // re-load the data file using the gateway-- reads from the home path
175        // now with attributes
176        H_HDF5Gate_Read_Raw(fileName+".h5")
177
178
179End
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199// passing null file string presents a dialog
200// these two procedures will use the full Xrefs, so they can write out full
201// files. They are, however, slow
202Proc Read_Nexus_Xref()
203        H_HDF5Gate_Read_Raw("")
204End
205
206Proc Write_Nexus_Xref()
207        H_HDF5Gate_Write_Raw()
208End
209
210//
211// writes out the contents of a data folder using the gateway
212// -- the HDF5___xref wave must be present at the top level for it to
213//    write out anything.
214//
215Proc H_HDF5Gate_Write_Raw(dfPath,filename)
216        String dfPath   = "root:"       // e.g., "root:FolderA" or ":"
217        String filename = "Test_VSANS_file.h5"
218//      Prompt dfPath, "Data Folder",popup,H_GetAList(4)       
219//      Prompt fileName, "File name"
220       
221//      DoPrompt "Folder and file to write", dfPath,fileName
222        // Check our work so far.
223        // If something prints, there was an error above.
224//      print H5GW_ValidateFolder(dfPath)
225
226// H5GW_WriteHDF5 calls the validate function, so no need to call it before
227        print H5GW_WriteHDF5(dfPath, filename)
228       
229        SetDataFolder root:
230End
231
232//
233// read in a data file using the gateway
234//
235// reads from the home path
236//
237Proc H_HDF5Gate_Read_Raw(file)
238        String file
239       
240//      NewDataFolder/O/S root:newdata
241        Print H5GW_ReadHDF5("", file)   // reads into current folder
242        SetDataFolder root:
243End
244
245
246
247// this is in PlotUtils, but duplicated here
248// utility used in the "PlotSmeared...() macros to get a list of data folders
249//
250//1:    Waves.
251//2:    Numeric variables.
252//3:    String variables.
253//4:    Data folders.
254Function/S H_GetAList(type)
255        Variable type
256       
257        SetDataFolder root:
258       
259        String objName,str=""
260        Variable index = 0
261        do
262                objName = GetIndexedObjName(":", type, index)
263                if (strlen(objName) == 0)
264                        break
265                endif
266                //Print objName
267                str += objName + ";"
268                index += 1
269        while(1)
270       
271        // remove myGlobals, Packages, etc. from the folder list
272        if(type==4)
273                str = RemoveFromList("myGlobals", str , ";" )
274                str = RemoveFromList("Packages", str, ";")
275        endif
276       
277        return(str)
278End
279
280//
281// after reading in a "partial" file using the gateway (to generate the xref)
282// Save the xref to disk (for later use)
283//
284Proc Save_HDF5___xref(dfPath, filename)
285        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
286        String filename = "HDF5___xref"
287
288        Save/T/P=home $(dfPath+":HDF5___xref") as "HDF5___xref.itx"
289
290//      Copy_HDF5___xref(dfPath, filename)
291       
292        SetDataFolder root:
293End
294
295//
296// after you've generated the HDF5___xref, load it in and copy it to
297// the necessary folder location.
298// - then all is set to *really* write out the file correctly, including the attributes
299//
300Proc Copy_HDF5___xref(dfPath, filename)
301        String dfPath   ="root:VSANS_file"              // e.g., "root:FolderA" or ":"
302        String filename = "HDF5___xref"
303
304        if(exists(filename) != 1)
305                //load it in   
306                LoadWave/T/P=home/O "HDF5___xref.itx"
307        endif
308       
309        Duplicate/O HDF5___xref, $(dfPath+":HDF5___xref")
310       
311        SetDataFolder root:
312End
313
314
315
316//////// testing procedures, may be of use, maybe not //////////////
317
318
319//////// Two procedures that test out Pete Jemain's HDF5Gateway
320//
321// This works fine, but it may not be terribly compatible with the way NICE will eventually
322// write out the data files. I'll have very little control over that and I'll need to cobble together
323// a bunch of fixes to cover up their mistakes.
324//
325// Using Nick Hauser's code as a starting point may be a lot more painful, but more flexible in the end.
326//
327// I'm completely baffled about what to do with attributes. Are they needed, is this the best way to deal
328// with them, do I care about reading them in, and if I do, why?
329//
330Proc H_HDF5Gate_WriteTest()
331
332        // create the folder structure
333        NewDataFolder/O/S root:mydata
334        NewDataFolder/O sasentry
335        NewDataFolder/O :sasentry:sasdata
336
337        // create the waves
338        Make/O :sasentry:sasdata:I0
339        Make/O :sasentry:sasdata:Q0
340
341        Make/O/N=0 Igor___folder_attributes
342        Make/O/N=0 :sasentry:Igor___folder_attributes
343        Make/O/N=0 :sasentry:sasdata:Igor___folder_attributes
344
345        // create the attributes
346        Note/K Igor___folder_attributes, "producer=IgorPro\rNX_class=NXroot"
347        Note/K :sasentry:Igor___folder_attributes, "NX_class=NXentry"
348        Note/K :sasentry:sasdata:Igor___folder_attributes, "NX_class=NXdata"
349        Note/K :sasentry:sasdata:I0, "units=1/cm\rsignal=1\rtitle=reduced intensity"
350        Note/K :sasentry:sasdata:Q0, "units=1/A\rtitle=|scattering vector|"
351
352        // create the cross-reference mapping
353        Make/O/T/N=(5,2) HDF5___xref
354        Edit/K=0 'HDF5___xref';DelayUpdate
355        HDF5___xref[0][1] = ":"
356        HDF5___xref[1][1] = ":sasentry"
357        HDF5___xref[2][1] = ":sasentry:sasdata"
358        HDF5___xref[3][1] = ":sasentry:sasdata:I0"
359        HDF5___xref[4][1] = ":sasentry:sasdata:Q0"
360        HDF5___xref[0][0] = "/"
361        HDF5___xref[1][0] = "/sasentry"
362        HDF5___xref[2][0] = "/sasentry/sasdata"
363        HDF5___xref[3][0] = "/sasentry/sasdata/I"
364        HDF5___xref[4][0] = "/sasentry/sasdata/Q"
365
366        // Check our work so far.
367        // If something prints, there was an error above.
368        print H5GW_ValidateFolder("root:mydata")
369
370        // set I0 and Q0 to your data
371
372        print H5GW_WriteHDF5("root:mydata", "mydata.h5")
373       
374        SetDataFolder root:
375End
376
377
378//
379// given a filename of a VSANS data file of the form
380// name.anything.any.other (any number of extensions is OK)
381// returns the name as a string without ANY ".fbdfasga" extension
382//
383// returns the input string if a "." can't be found (maybe it wasn't there)
384//
385Function/S H_RemoveDotExtension(item)
386        String item
387        String invalid = item   //
388        Variable num=-1
389       
390        //find the "dot"
391        String runStr=""
392        Variable pos = strsearch(item,".",0)
393        if(pos == -1)
394                //"dot" not found
395                return (invalid)
396        else
397                //found, get all of the characters preceeding it
398                runStr = item[0,pos-1]
399                return (runStr)
400        Endif
401End
402
403
404//
405// Writing attributes to a group and to a dataset.
406// example from the WM documentation for HDF5SaveData
407//
408Function DemoAttributes(w)
409        Wave w
410
411        Variable result = 0     // 0 means no error
412       
413        // Create file
414        Variable fileID
415        HDF5CreateFile/P=home /O /Z fileID as "Test.h5"
416        if (V_flag != 0)
417                Print "HDF5CreateFile failed"
418                return -1
419        endif
420
421        // Write an attribute to the root group
422        Make /FREE /T /N=1 groupAttribute = "This is a group attribute"
423        HDF5SaveData /A="GroupAttribute" groupAttribute, fileID, "/"
424       
425        // Save wave as dataset
426        HDF5SaveData /O /Z w, fileID    // Uses wave name as dataset name
427        if (V_flag != 0)
428                Print "HDF5SaveData failed"
429                result = -1
430        endif
431
432        // Write an attribute to the dataset
433        Make /FREE /T /N=1 datasetAttribute = "This is a dataset attribute"
434        String datasetName = NameOfWave(w)
435        HDF5SaveData /A="DatasetAttribute" datasetAttribute, fileID, datasetName
436
437        HDF5CloseFile fileID
438       
439        return result
440End     // The attribute waves are automatically killed since they are free waves
441
442
443
444//     
445//Function H_Test_HDFWriteTrans(fname,val)
446//      String fname
447//      Variable val
448//     
449//     
450//      String str
451//      PathInfo home
452//      str = S_path
453//     
454//      H_WriteTransmissionToHeader(str+fname,val)
455//     
456//      return(0)
457//End
458//
459//Function H_WriteTransmissionToHeader(fname,trans)
460//      String fname
461//      Variable trans
462//     
463//      Make/O/D/N=1 wTmpWrite
464//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
465//      String varName = "TRNS"
466//      wTmpWrite[0] = trans //
467//
468//      variable err
469//      err = HDFWrite_Wave(fname, groupName, varName, wTmpWrite)
470//      KillWaves wTmpWrite
471//     
472//      //err not handled here
473//             
474//      return(0)
475//End
476
477
478
479Function H_Test_ListAttributes(fname,groupName)
480        String fname,groupName
481        Variable trans
482       
483//      Make/O/D/N=1 wTmpWrite
484//      String groupName = "/Sample"    //      /Run1/Sample becomes groupName /Run1/Run1/Sample
485//      String varName = "TRNS"
486//      wTmpWrite[0] = trans //
487        String str
488        PathInfo home
489        str = S_path
490       
491        variable err
492        err = H_HDF_ListAttributes(str+fname, groupName)
493       
494        //err not handled here
495               
496        return(0)
497End
498
499Function H_HDF_ListAttributes(fname, groupName)
500        String fname, groupName
501       
502        variable err=0, fileID,groupID
503        String cDF = getDataFolder(1), temp
504        String NXentry_name, attrValue=""
505       
506        STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
507        InitHDF5DataInfo(di)    // Initialize structure.
508       
509        try     
510                HDF5OpenFile /Z fileID  as fname  //open file read-write
511                if(!fileID)
512                        err = 1
513                        abort "HDF5 file does not exist"
514                endif
515               
516                HDF5OpenGroup /Z fileID , groupName, groupID
517
518        //      (QUOKKA) !! At the moment, there is no entry for sample thickness in our data file
519        //      therefore create new HDF5 group to enable write / patch command
520        //      comment out the following group creation once thickness appears in revised file
521       
522                if(!groupID)
523                        HDF5CreateGroup /Z fileID, groupName, groupID
524                        //err = 1
525                        //abort "HDF5 group does not exist"
526                else
527
528//                      HDF5AttributeInfo(fileID, "/", 1, "file_name", 0, di)
529                        HDF5AttributeInfo(fileID, "/", 1, "NeXus_version", 0, di)
530                        Print di
531
532//                      see the HDF5 Browser  for how to get the actual <value> of the attribute. See GetPreviewString in
533//        or in FillGroupAttributesList or in FillDatasetAttributesList (from FillLists)
534//                      it seems to be ridiculously complex to get such a simple bit of information - the HDF5BrowserData STRUCT
535//                      needs to be filled first. Ugh.
536                        attrValue = GetPreviewString(fileID, 1, di, "/entry", "cucumber")
537                        Print "attrValue = ",attrValue
538                       
539                       
540                        //get attributes and save them
541                        HDF5ListAttributes/TYPE=1 /Z fileID, groupName          //TYPE=1 means that we're referencing a group, not a dataset
542                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
543                       
544                        // passing the groupID works too, then the group name is not needed                     
545                        HDF5ListAttributes/TYPE=1 /Z groupID, "."               //TYPE=1 means that we're referencing a group, not a dataset
546                        Print "S_HDF5ListAttributes = ", S_HDF5ListAttributes
547                endif
548        catch
549
550                // catch any aborts here
551               
552        endtry
553       
554        if(groupID)
555                HDF5CloseGroup /Z groupID
556        endif
557       
558        if(fileID)
559                HDF5CloseFile /Z fileID
560        endif
561
562        setDataFolder $cDF
563        return err
564end
565
566
567/////////////// end of the testing procedures ////////////////
Note: See TracBrowser for help on using the repository browser.