source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/V_HDF5_RW_Utils.ipf @ 981

Last change on this file since 981 was 981, checked in by srkline, 7 years ago

more changes, filling in functionality of the initial data correction

File size: 18.8 KB
Line 
1#pragma rtGlobals=3             // Use modern global access method and strict wave access.
2
3//
4// The base functions for R/W from HDF5 files.
5// All of the specific "get" and "write" functions call these base functions which are responsible
6// for all of the open/close mechanics.
7//
8// These VSANS file-specific functions are in:
9// V_HDF5_Read.ipf
10//               and
11// V_HDF5_Write.ipf
12//
13
14// data is read into:
15//      NewDataFolder/O/S root:Packages:NIST:VSANS:RawVSANS
16// so that the data folders of the raw data won't be lying around on the top level, looking like
17// 1D data sets and interfering.
18
19
20// the base data folder path where the raw data is loaded
21Strconstant ksBaseDFPath = "root:Packages:NIST:VSANS:RawVSANS:"
22
23
24// passing null file string presents a dialog
25Macro LoadFakeDIVData()
26        V_LoadHDF5Data("","DIV")
27End
28
29// passing null file string presents a dialog
30Proc Read_HDF5_Raw_No_Attributes()
31        V_LoadHDF5Data("","RAW")
32End
33
34
35// TODO:
36//  x- move the initialization of the raw data folder to be in the as-yet unwritten initialization routine for
37// reduction. be sure that it's duplicated in the VCALC initialization too.
38// -- as needed, get rid of the FAKE redimension of the data from 3D->2D and from 128x128 to something else for VSANS
39//    This is a fake since I don't have anything close to correct fake data yet. (1/29/16)
40//
41// TODO: -- is there an extra "entry" heading? Am I adding this by mistake by setting base_name="entry" for RAW data?
42//
43Function V_LoadHDF5Data(file,folder)
44        String file,folder
45
46        String base_name
47        SetDataFolder $("root:Packages:NIST:VSANS:"+folder)
48//      SetDataFolder root:
49        if(cmpstr(folder,"RAW")==0)
50                base_name="entry"
51        else
52                base_name="entry"               //TODO -- remove this / change behavior in V_LoadHDF5_NoAtt()
53        endif
54       
55        Variable err= V_LoadHDF5_NoAtt(file,base_name)  // reads into current folder
56       
57        // if RAW data, then generate the errors and linear data copy
58        // do this 9x
59        string tmpStr = "root:Packages:NIST:VSANS:RAW:entry:entry:instrument:"
60        if(cmpstr(folder,"RAW")==0)
61                V_MakeDataError(tmpStr+"detector_B")
62                V_MakeDataError(tmpStr+"detector_MB")
63                V_MakeDataError(tmpStr+"detector_MT")
64                V_MakeDataError(tmpStr+"detector_ML")
65                V_MakeDataError(tmpStr+"detector_MR")
66                V_MakeDataError(tmpStr+"detector_FB")
67                V_MakeDataError(tmpStr+"detector_FT")
68                V_MakeDataError(tmpStr+"detector_FL")
69                V_MakeDataError(tmpStr+"detector_FR")
70
71// TODO -- once I get "real" data, get rid of this call to force the data to be proper dimensions.
72                V_RedimFakeData()
73//
74// TODO -- for the "real" data, may need a step in here to convert integer detector data to DP, or I'll
75//          get really odd results from the calculations, and may not even notice.
76
77// TODO
78//  -- get rid of these fake calibration waves as "real" ones are filled in
79                Execute "MakeFakeCalibrationWaves()"
80//              fMakeFakeCalibrationWaves()             //skips the alert
81               
82                               
83/// END FAKE DATA CORRECTIONS           
84               
85        endif
86       
87        SetDataFolder root:
88        return(err)
89End
90
91//
92// TODO -- this is all FAKED since all the data arrays are (1,128,128)
93// I'm intentionally using the wrong number of pixels so I'm more likely to go back and get rid of this later.
94//
95// the SetScale parts may be useful later.
96//
97Function V_RedimFakeData()
98       
99                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_B
100                Wave det_B=data
101                Redimension/N=(320,320)/E=1 det_B
102                det_B = p+q+2
103               
104                Variable ctr=20,npix=128
105                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MT
106                Wave det_MT=data
107                Redimension/N=(npix,48)/E=1 det_MT             
108                SetScale/I x -npix/2,npix/2,"",det_MT
109                SetScale/I y ctr,ctr+48,"",det_MT
110//              det_mt[][20] = 50
111        det_MT *= 10
112        det_MT += 2
113
114                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MB
115                Wave det_MB=data
116                Redimension/N=(npix,48)/E=1 det_MB             
117                SetScale/I x -npix/2,npix/2,"",det_MB
118                SetScale/I y -ctr,-ctr-48,"",det_MB
119        det_MB *= 5
120        det_MB += 2
121               
122                ctr=30
123                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_ML
124                Wave det_ML=data
125                Redimension/N=(48,npix)/E=1 det_ML             
126                SetScale/I x -ctr-48,-ctr,"",det_ML
127                SetScale/I y -npix/2,npix/2,"",det_ML
128        det_ML *= 2
129        det_ML += 2
130       
131                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MR
132                Wave det_MR=data
133                Redimension/N=(48,npix)/E=1 det_MR             
134                SetScale/I x ctr,ctr+48,"",det_MR
135                SetScale/I y -npix/2,npix/2,"",det_MR
136        det_MR +=2
137               
138                ctr=30
139                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FT
140                Wave det_FT=data
141                Redimension/N=(npix,48)/E=1 det_FT             
142                SetScale/I x -npix/2,npix/2,"",det_FT
143                SetScale/I y ctr,ctr+48,"",det_FT
144
145                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FB
146                Wave det_FB=data
147                Redimension/N=(npix,48)/E=1 det_FB             
148                SetScale/I x -npix/2,npix/2,"",det_FB
149                SetScale/I y -ctr,-ctr-48,"",det_FB
150               
151                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FL
152                Wave det_FL=data
153                Redimension/N=(48,npix)/E=1 det_FL             
154                SetScale/I x -ctr-48,-ctr,"",det_FL
155                SetScale/I y -npix/2,npix/2,"",det_FL
156               
157                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FR
158                Wave det_FR=data
159                Redimension/N=(48,npix)/E=1 det_FR             
160                SetScale/I x ctr,ctr+48,"",det_FR
161                SetScale/I y -npix/2,npix/2,"",det_FR
162
163// get rid of zeros
164                det_FL += 2
165                det_FR += 2
166                det_FT += 2
167                det_FB += 2
168
169        return(0)
170End
171
172
173// This loads for speed, since loading the attributes takes a LOT of time.
174//
175// this will load in the whole HDF file all at once.
176// Attributes are NOT loaded at all.
177//
178// TODO:
179// -x remove the P=home restriction top make this more generic (replaced with catPathName from PickPath)
180// -- get rid of bits leftover here that I don't need
181// -- be sure I'm using all of the correct flags in the HDF5LoadGroup operation
182// -- settle on how the base_name is to be used. "entry" for the RAW, fileName for the "rawVSANS"?
183//
184// passing in "" for base_name will take the name from the file name as selected
185//
186Function V_LoadHDF5_NoAtt(fileName,base_name)
187        String fileName, base_name
188       
189//      if ( ParamIsDefault(hdf5Path) )
190//              hdf5Path = "/"
191//      endif
192
193        String hdf5path = "/"           //always read from the top
194        String status = ""
195
196        Variable fileID = 0
197//      HDF5OpenFile/R/P=home/Z fileID as fileName              //read file from home directory?
198        HDF5OpenFile/R/P=catPathName/Z fileID as fileName
199        if (V_Flag != 0)
200                return 0
201        endif
202
203        String/G root:file_path = S_path
204        String/G root:file_name = S_FileName
205       
206        if ( fileID == 0 )
207                Print fileName + ": could not open as HDF5 file"
208                return (0)
209        endif
210       
211//s_tic()               //fast
212       
213        SVAR tmpStr=root:file_name
214        fileName=tmpStr         //SRK - in case the file was chosen from a dialog, I'll need access to the name later
215       
216        //   read the data (too bad that HDF5LoadGroup does not read the attributes)
217        if(cmpstr(base_name,"") == 0)
218                base_name = StringFromList(0,FileName,".")
219        endif
220        //base_name = "entry"
221       
222        HDF5LoadGroup/Z/L=7/O/R/T=$base_name  :, fileID, hdf5Path               //      recursive
223        if ( V_Flag != 0 )
224                Print fileName + ": could not open as HDF5 file"
225                setdatafolder root:
226                return (0)
227        endif
228
229        HDF5CloseFile fileID
230       
231//s_toc()
232        return(0)
233end     
234
235
236// read a single real value
237// - fname passed in is the full path to the file on disk
238// - path is the path to the value in the HDF tree
239//
240// check to see if the value exists (It will be a wave)
241// -- if it does, return the value from the local folder
242// -- if not, read the file in, then return the value
243//
244// TODO:
245// currently, the work folders have the following path - so passing in "RAW" as fname
246// will take some re-configuring.
247//  root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FL:distance
248// -- be sure this read from work folders is not broken in the future, and is passed to ALL of the
249//    top-level R/W routines. (Write is necessary ONLY for SIM data files. Patch is direct to disk.)
250Function V_getRealValueFromHDF5(fname,path)
251        String fname,path
252
253        String folderStr=""
254        Variable valExists=0
255       
256        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
257
258// check for a work folder first (note that "entry" is doubled)
259        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
260                Wave/Z w = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
261                return(w[0])
262        endif
263       
264        if(Exists(ksBaseDFPath+folderStr+":"+path))
265                valExists=1
266        endif
267       
268        if(!valExists)
269                //then read in the file
270                V_LoadHDF5_NoAtt(fname,"")
271        endif
272
273// this should exist now - if not, I need to see the error
274        Wave/Z w = $(ksBaseDFPath+folderStr+":"+path)
275       
276        if(WaveExists(w))
277                return(w[0])
278        else
279                return(-999999)
280        endif   
281End
282
283// Returns a wave reference, not just a single value
284// ---then you pick what you need from the wave
285//
286// - fname passed in is the full path to the file on disk
287// - path is the path to the value in the HDF tree
288//
289// check to see if the value exists (It will be a wave)
290// -- if it does, return the value from the local folder
291// -- if not, read the file in, then return the value
292//
293Function/WAVE V_getRealWaveFromHDF5(fname,path)
294        String fname,path
295
296        String folderStr=""
297        Variable valExists=0
298       
299        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
300
301// check for a work folder first (note that "entry" is doubled)
302        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
303                Wave wOut = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
304                return wOut
305        endif
306               
307        if(Exists(ksBaseDFPath+folderStr+":"+path))
308                valExists=1
309        endif
310       
311        if(!valExists)
312                //then read in the file
313                V_LoadHDF5_NoAtt(fname,"")
314        endif
315
316// this should exist now - if not, I need to see the error
317        Wave wOut = $(ksBaseDFPath+folderStr+":"+path)
318       
319        return wOut
320       
321End
322
323// Returns a wave reference, not just a single value
324// ---then you pick what you need from the wave
325//
326// - fname passed in is the full path to the file on disk
327// - path is the path to the value in the HDF tree
328//
329// check to see if the value exists (It will be a wave)
330// -- if it does, return the value from the local folder
331// -- if not, read the file in, then return the value
332//
333Function/WAVE V_getTextWaveFromHDF5(fname,path)
334        String fname,path
335
336        String folderStr=""
337        Variable valExists=0
338       
339        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
340
341// check for a work folder first (note that "entry" is doubled)
342        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
343                Wave/T wOut = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
344                return wOut
345        endif
346       
347        if(Exists(ksBaseDFPath+folderStr+":"+path))
348                valExists=1
349        endif
350       
351        if(!valExists)
352                //then read in the file
353                V_LoadHDF5_NoAtt(fname,"")
354        endif
355
356// this should exist now - if not, I need to see the error
357        Wave/T wOut = $(ksBaseDFPath+folderStr+":"+path)
358       
359        return wOut
360       
361End
362
363
364//
365//   TODO
366// depricated? in HDF5 - store all of the values as real?
367// Igor sees no difference in real and integer variables (waves are different)
368// BUT-- Igor 7 will have integer variables
369//
370// truncate to integer before returning??
371//
372//  TODO
373// write a "getIntegerWave" function??
374//
375//////  integer values
376// reads 32 bit integer
377Function V_getIntegerFromHDF5(fname,path)
378        String fname                            //full path+name
379        String path                             //path to the hdf5 location
380       
381        Variable val = V_getRealValueFromHDF5(fname,path)
382       
383        val = round(val)
384        return(val)
385End
386
387
388// read a single string
389// - fname passed in is the full path to the file on disk
390// - path is the path to the value in the HDF tree
391// - num is the number of characters in the VAX string
392// check to see if the value exists (It will be a wave)
393// -- if it does, return the value from the local folder
394// -- if not, read the file in, then return the value
395//
396// TODO -- string could be checked for length, but returned right or wrong
397//
398Function/S V_getStringFromHDF5(fname,path,num)
399        String fname,path
400        Variable num
401
402        String folderStr=""
403        Variable valExists=0
404       
405        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
406
407// check for a work folder first (note that "entry" is doubled)
408        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
409                Wave/Z/T tw = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
410                return(tw[0])
411        endif
412       
413        if(Exists(ksBaseDFPath+folderStr+":"+path))
414                valExists=1
415        endif
416       
417        if(!valExists)
418                //then read in the file
419                V_LoadHDF5_NoAtt(fname,"")
420        endif
421
422// this should exist now - if not, I need to see the error
423        Wave/T/Z tw = $(ksBaseDFPath+folderStr+":"+path)
424       
425        if(WaveExists(tw))
426       
427        //      if(strlen(tw[0]) != num)
428        //              Print "string is not the specified length"
429        //      endif
430               
431                return(tw[0])
432        else
433                return("The specified wave does not exist: " + path)
434        endif
435End
436
437
438
439///////////////////////////////
440
441//
442//Write Wave 'wav' to hdf5 file 'fname'
443//Based on code from ANSTO (N. Hauser. nha 8/1/09)
444//
445// TODO:
446// -- figure out if this will write in the native format of the
447//     wave as passed in, or if it will only write as DP.
448// -- do I need to write separate functions for real, integer, etc.?
449//     
450// -x change the /P=home to the user-defined data path (catPathName)           
451//
452Function V_WriteWaveToHDF(fname, groupName, varName, wav)
453        String fname, groupName, varName
454        Wave wav
455       
456        variable err=0, fileID,groupID
457        String cDF = getDataFolder(1), temp
458        String NXentry_name
459       
460        try     
461                HDF5OpenFile/P=catPathName /Z fileID  as fname  //open file read-write
462                if(!fileID)
463                        err = 1
464                        abort "HDF5 file does not exist"
465                endif
466               
467                //get the NXentry node name
468                HDF5ListGroup /TYPE=1 fileID, "/"
469                //remove trailing ; from S_HDF5ListGroup
470               
471                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
472               
473                NXentry_name = S_HDF5ListGroup
474                NXentry_name = ReplaceString(";",NXentry_name,"")
475                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
476                        err = 1
477                        abort "More than one entry under the root node. Ambiguous"
478                endif
479                //concatenate NXentry node name and groupName   
480                // SRK - NOV2015 - dropped this and require the full group name passed in
481//              groupName = "/" + NXentry_name + groupName
482                Print "groupName = ",groupName
483                HDF5OpenGroup /Z fileID , groupName, groupID
484
485                if(!groupID)
486                // don't create the group it the name isn't right -- throw up an error
487                        //HDF5CreateGroup /Z fileID, groupName, groupID
488                        err = 1
489                        HDF5CloseFile /Z fileID
490                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
491                        return(err)
492                else
493                        // get attributes and save them
494                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
495                        //Wave attributes = S_HDF5ListAttributes
496                endif
497       
498                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
499                if (V_flag != 0)
500                        err = 1
501                        abort "Cannot save wave to HDF5 dataset " + varName
502                endif   
503               
504               
505                //attributes - something could be added here as optional parameters and flagged
506//              String attributes = "units"
507//              Make/O/T/N=1 tmp
508//              tmp[0] = "dimensionless"
509//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
510//              if (V_flag != 0)
511//                      err = 1
512//                      abort "Cannot save attributes to HDF5 dataset"
513//              endif   
514        catch
515
516        endtry
517
518// it is not necessary to close the group here. HDF5CloseFile will close the group as well     
519        if(groupID)
520                HDF5CloseGroup /Z groupID
521        endif
522       
523        if(fileID)
524                HDF5CloseFile /Z fileID
525        endif
526
527        setDataFolder $cDF
528        return err
529end
530
531//Write Wave 'wav' to hdf5 file 'fname'
532//Based on code from ANSTO (N. Hauser. nha 8/1/09)
533//
534// TODO
535//
536// -x change the /P=home to the user-defined data path (catPathName)           
537//
538Function V_WriteTextWaveToHDF(fname, groupName, varName, wav)
539        String fname, groupName, varName
540        Wave/T wav
541       
542        variable err=0, fileID,groupID
543        String cDF = getDataFolder(1), temp
544        String NXentry_name
545       
546        try     
547                HDF5OpenFile/P=catPathName /Z fileID  as fname  //open file read-write
548                if(!fileID)
549                        err = 1
550                        abort "HDF5 file does not exist"
551                endif
552               
553                //get the NXentry node name
554                HDF5ListGroup /TYPE=1 fileID, "/"
555                //remove trailing ; from S_HDF5ListGroup
556               
557                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
558               
559                NXentry_name = S_HDF5ListGroup
560                NXentry_name = ReplaceString(";",NXentry_name,"")
561                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
562                        err = 1
563                        abort "More than one entry under the root node. Ambiguous"
564                endif
565
566                //concatenate NXentry node name and groupName
567                // SRK - NOV2015 - dropped this and require the full group name passed in
568//              groupName = "/" + NXentry_name + groupName
569                Print "groupName = ",groupName
570
571                HDF5OpenGroup /Z fileID , groupName, groupID
572
573                if(!groupID)
574                // don't create the group it the name isn't right -- throw up an error
575                        //HDF5CreateGroup /Z fileID, groupName, groupID
576                        err = 1
577                        HDF5CloseFile /Z fileID
578                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
579                        return(err)
580                else
581                        // get attributes and save them
582                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
583                        //Wave attributes = S_HDF5ListAttributes
584                endif
585       
586                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
587                if (V_flag != 0)
588                        err = 1
589                        abort "Cannot save wave to HDF5 dataset " + varName
590                endif   
591               
592               
593                //attributes - something could be added here as optional parameters and flagged
594//              String attributes = "units"
595//              Make/O/T/N=1 tmp
596//              tmp[0] = "dimensionless"
597//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
598//              if (V_flag != 0)
599//                      err = 1
600//                      abort "Cannot save attributes to HDF5 dataset"
601//              endif   
602        catch
603
604        endtry
605       
606        if(groupID)
607                HDF5CloseGroup /Z groupID
608        endif
609       
610        if(fileID)
611                HDF5CloseFile /Z fileID
612        endif
613
614        setDataFolder $cDF
615        return err
616end
617
618//////////////////////////////
619//////////////////////////////
620//////////////////////////////
621
622Function V_KillNamedDataFolder(fname)
623        String fname
624       
625        Variable err=0
626       
627        String folderStr = V_GetFileNameFromPathNoSemi(fname)
628        folderStr = V_RemoveDotExtension(folderStr)
629       
630        KillDataFolder/Z $(ksBaseDFPath+folderStr)
631        err = V_flag
632       
633        return(err)
634end
635
636//given a filename of a SANS data filename of the form
637// name.anything
638//returns the name as a string without the ".fbdfasga" extension
639//
640// returns the input string if a"." can't be found (maybe it wasn't there"
641Function/S V_RemoveDotExtension(item)
642        String item
643        String invalid = item   //
644        Variable num=-1
645       
646        //find the "dot"
647        String runStr=""
648        Variable pos = strsearch(item,".",0)
649        if(pos == -1)
650                //"dot" not found
651                return (invalid)
652        else
653                //found, get all of the characters preceeding it
654                runStr = item[0,pos-1]
655                return (runStr)
656        Endif
657End
658
659//returns a string containing filename (WITHOUT the ;vers)
660//the input string is a full path to the file (Mac-style, still works on Win in IGOR)
661//with the folders separated by colons
662//
663// called by MaskUtils.ipf, ProtocolAsPanel.ipf, WriteQIS.ipf
664//
665Function/S V_GetFileNameFromPathNoSemi(fullPath)
666        String fullPath
667       
668        Variable offset1,offset2
669        String filename=""
670        //String PartialPath
671        offset1 = 0
672        do
673                offset2 = StrSearch(fullPath, ":", offset1)
674                if (offset2 == -1)                              // no more colons ?
675                        fileName = FullPath[offset1,strlen(FullPath) ]
676                        //PartialPath = FullPath[0, offset1-1]
677                        break
678                endif
679                offset1 = offset2+1
680        while (1)
681       
682        //remove version number from name, if it's there - format should be: filename;N
683        filename =  StringFromList(0,filename,";")              //returns null if error
684       
685        Return filename
686End
Note: See TracBrowser for help on using the repository browser.