source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/V_HDF5_RW_Utils.ipf @ 979

Last change on this file since 979 was 979, checked in by srkline, 7 years ago

more additions to start the work file flow of converting RAW folder to a WORK folder. Raw_to_Work will be the function that sequentially applies the corrections. All corrections can be turned on/off with preferences.

File size: 18.0 KB
Line 
1#pragma rtGlobals=3             // Use modern global access method and strict wave access.
2
3//
4// The base functions for R/W from HDF5 files.
5// All of the specific "get" and "write" functions call these base functions which are responsible
6// for all of the open/close mechanics.
7//
8// These VSANS file-specific functions are in:
9// V_HDF5_Read.ipf
10//               and
11// V_HDF5_Write.ipf
12//
13
14// data is read into:
15//      NewDataFolder/O/S root:Packages:NIST:VSANS:RawVSANS
16// so that the data folders of the raw data won't be lying around on the top level, looking like
17// 1D data sets and interfering.
18
19
20// the base data folder path where the raw data is loaded
21Strconstant ksBaseDFPath = "root:Packages:NIST:VSANS:RawVSANS:"
22
23
24
25// passing null file string presents a dialog
26Proc Read_HDF5_Raw_No_Attributes()
27        V_LoadHDF5Data("","RAW")
28End
29
30
31// TODO:
32//  x- move the initializtion of the raw data folder to be in the as-yet unwritten initialization routine for
33// reduction. be sure that it's duplicated in the VCALC initialization too.
34// -- as needed, get rid of the FAKE redimension of the data from 3D->2D and from 128x128 to something else for VSANS
35//    This is a fake since I don't have anything close to correct fake data yet. (1/29/16)
36//
37Function V_LoadHDF5Data(file,folder)
38        String file,folder
39
40        String base_name
41        SetDataFolder $("root:Packages:NIST:VSANS:"+folder)
42//      SetDataFolder root:
43        if(cmpstr(folder,"RAW")==0)
44                base_name="entry"
45        else
46                base_name=""
47        endif
48       
49        Variable err= V_LoadHDF5_NoAtt(file,base_name)  // reads into current folder
50       
51        // if RAW data, then generate the errors and linear data copy
52        // do this 9x
53        string tmpStr = "root:Packages:NIST:VSANS:RAW:entry:entry:instrument:"
54        if(cmpstr(folder,"RAW")==0)
55                V_MakeDataError(tmpStr+"detector_B")
56                V_MakeDataError(tmpStr+"detector_MB")
57                V_MakeDataError(tmpStr+"detector_MT")
58                V_MakeDataError(tmpStr+"detector_ML")
59                V_MakeDataError(tmpStr+"detector_MR")
60                V_MakeDataError(tmpStr+"detector_FB")
61                V_MakeDataError(tmpStr+"detector_FT")
62                V_MakeDataError(tmpStr+"detector_FL")
63                V_MakeDataError(tmpStr+"detector_FR")
64
65// TODO -- once I get "real" data, get rid of this call to force the data to be proper dimensions.
66                V_RedimFakeData()
67               
68/// END FAKE DATA CORRECTIONS           
69               
70        endif
71       
72        SetDataFolder root:
73        return(err)
74End
75
76//
77// TODO -- this is all FAKED since all the data arrays are (1,128,128)
78// I'm intentionally using the wrong number of pixels so I'm more likely to go back and get rid of this later.
79//
80// the SetScale parts may be useful later.
81//
82Function V_RedimFakeData()
83       
84                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_B
85                Wave det_B=data
86                Redimension/N=(300,300)/E=1 det_B
87               
88                Variable ctr=20,npix=100
89                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MT
90                Wave det_MT=data
91                Redimension/N=(npix,48)/E=1 det_MT             
92                SetScale/I x -npix/2,npix/2,"",det_MT
93                SetScale/I y ctr,ctr+48,"",det_MT
94//              det_mt[][20] = 50
95        det_MT *= 10
96        det_MT += 2
97
98                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MB
99                Wave det_MB=data
100                Redimension/N=(npix,48)/E=1 det_MB             
101                SetScale/I x -npix/2,npix/2,"",det_MB
102                SetScale/I y -ctr,-ctr-48,"",det_MB
103        det_MB *= 5
104        det_MB += 2
105               
106                ctr=30
107                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_ML
108                Wave det_ML=data
109                Redimension/N=(48,npix)/E=1 det_ML             
110                SetScale/I x -ctr-48,-ctr,"",det_ML
111                SetScale/I y -npix/2,npix/2,"",det_ML
112        det_ML *= 2
113        det_ML += 2
114       
115                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MR
116                Wave det_MR=data
117                Redimension/N=(48,npix)/E=1 det_MR             
118                SetScale/I x ctr,ctr+48,"",det_MR
119                SetScale/I y -npix/2,npix/2,"",det_MR
120        det_MR +=2
121               
122                ctr=30
123                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FT
124                Wave det_FT=data
125                Redimension/N=(npix,48)/E=1 det_FT             
126                SetScale/I x -npix/2,npix/2,"",det_FT
127                SetScale/I y ctr,ctr+48,"",det_FT
128
129                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FB
130                Wave det_FB=data
131                Redimension/N=(npix,48)/E=1 det_FB             
132                SetScale/I x -npix/2,npix/2,"",det_FB
133                SetScale/I y -ctr,-ctr-48,"",det_FB
134               
135                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FL
136                Wave det_FL=data
137                Redimension/N=(48,npix)/E=1 det_FL             
138                SetScale/I x -ctr-48,-ctr,"",det_FL
139                SetScale/I y -npix/2,npix/2,"",det_FL
140               
141                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FR
142                Wave det_FR=data
143                Redimension/N=(48,npix)/E=1 det_FR             
144                SetScale/I x ctr,ctr+48,"",det_FR
145                SetScale/I y -npix/2,npix/2,"",det_FR
146
147
148        return(0)
149End
150
151
152// This loads for speed, since loading the attributes takes a LOT of time.
153//
154// this will load in the whole HDF file all at once.
155// Attributes are NOT loaded at all.
156//
157// TODO: remove the P=home restriction top make this more generic
158// -- get rid of bits leftover here that I don't need
159// -- be sure I'm using all of the correct flags in the HDF5LoadGroup operation
160// -- settle on how the base_name is to be used. "entry" for the RAW, fileName for the "rawVSANS"?
161//
162// passing in "" for base_name will take the name from the file name as selected
163//
164Function V_LoadHDF5_NoAtt(fileName,base_name)
165        String fileName, base_name
166       
167//      if ( ParamIsDefault(hdf5Path) )
168//              hdf5Path = "/"
169//      endif
170
171        String hdf5path = "/"           //always read from the top
172        String status = ""
173
174        Variable fileID = 0
175        HDF5OpenFile/R/P=home/Z fileID as fileName              //read file from home directory?
176//      HDF5OpenFile/R/P=catPathName/Z fileID as fileName
177        if (V_Flag != 0)
178                return 0
179        endif
180
181        String/G root:file_path = S_path
182        String/G root:file_name = S_FileName
183       
184        if ( fileID == 0 )
185                Print fileName + ": could not open as HDF5 file"
186                return (0)
187        endif
188       
189//s_tic()               //fast
190       
191        SVAR tmpStr=root:file_name
192        fileName=tmpStr         //SRK - in case the file was chosen from a dialog, I'll need access to the name later
193       
194        //   read the data (too bad that HDF5LoadGroup does not read the attributes)
195        if(cmpstr(base_name,"") == 0)
196                base_name = StringFromList(0,FileName,".")
197        endif
198        //base_name = "entry"
199       
200        HDF5LoadGroup/Z/L=7/O/R/T=$base_name  :, fileID, hdf5Path               //      recursive
201        if ( V_Flag != 0 )
202                Print fileName + ": could not open as HDF5 file"
203                setdatafolder root:
204                return (0)
205        endif
206
207        HDF5CloseFile fileID
208       
209//s_toc()
210        return(0)
211end     
212
213
214// read a single real value
215// - fname passed in is the full path to the file on disk
216// - path is the path to the value in the HDF tree
217//
218// check to see if the value exists (It will be a wave)
219// -- if it does, return the value from the local folder
220// -- if not, read the file in, then return the value
221//
222// TODO:
223// currently, the work folders have the following path - so passing in "RAW" as fname
224// will take some re-configuring.
225//  root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FL:distance
226// -- be sure this read from work folders is not broken in the future, and is passed to ALL of the
227//    top-level R/W routines. (Write is necessary ONLY for SIM data files. Patch is direct to disk.)
228Function V_getRealValueFromHDF5(fname,path)
229        String fname,path
230
231        String folderStr=""
232        Variable valExists=0
233       
234        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
235
236// check for a work folder first (note that "entry" is doubled)
237        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
238                Wave/Z w = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
239                return(w[0])
240        endif
241       
242        if(Exists(ksBaseDFPath+folderStr+":"+path))
243                valExists=1
244        endif
245       
246        if(!valExists)
247                //then read in the file
248                V_LoadHDF5_NoAtt(fname,"")
249        endif
250
251// this should exist now - if not, I need to see the error
252        Wave/Z w = $(ksBaseDFPath+folderStr+":"+path)
253       
254        if(WaveExists(w))
255                return(w[0])
256        else
257                return(-999999)
258        endif   
259End
260
261// Returns a wave reference, not just a single value
262// ---then you pick what you need from the wave
263//
264// - fname passed in is the full path to the file on disk
265// - path is the path to the value in the HDF tree
266//
267// check to see if the value exists (It will be a wave)
268// -- if it does, return the value from the local folder
269// -- if not, read the file in, then return the value
270//
271Function/WAVE V_getRealWaveFromHDF5(fname,path)
272        String fname,path
273
274        String folderStr=""
275        Variable valExists=0
276       
277        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
278
279// check for a work folder first (note that "entry" is doubled)
280        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
281                Wave wOut = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
282                return wOut
283        endif
284               
285        if(Exists(ksBaseDFPath+folderStr+":"+path))
286                valExists=1
287        endif
288       
289        if(!valExists)
290                //then read in the file
291                V_LoadHDF5_NoAtt(fname,"")
292        endif
293
294// this should exist now - if not, I need to see the error
295        Wave wOut = $(ksBaseDFPath+folderStr+":"+path)
296       
297        return wOut
298       
299End
300
301// Returns a wave reference, not just a single value
302// ---then you pick what you need from the wave
303//
304// - fname passed in is the full path to the file on disk
305// - path is the path to the value in the HDF tree
306//
307// check to see if the value exists (It will be a wave)
308// -- if it does, return the value from the local folder
309// -- if not, read the file in, then return the value
310//
311Function/WAVE V_getTextWaveFromHDF5(fname,path)
312        String fname,path
313
314        String folderStr=""
315        Variable valExists=0
316       
317        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
318
319// check for a work folder first (note that "entry" is doubled)
320        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
321                Wave/T wOut = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
322                return wOut
323        endif
324       
325        if(Exists(ksBaseDFPath+folderStr+":"+path))
326                valExists=1
327        endif
328       
329        if(!valExists)
330                //then read in the file
331                V_LoadHDF5_NoAtt(fname,"")
332        endif
333
334// this should exist now - if not, I need to see the error
335        Wave/T wOut = $(ksBaseDFPath+folderStr+":"+path)
336       
337        return wOut
338       
339End
340
341
342//
343//   TODO
344// depricated? in HDF5 - store all of the values as real?
345// Igor sees no difference in real and integer variables (waves are different)
346// BUT-- Igor 7 will have integer variables
347//
348// truncate to integer before returning??
349//
350//  TODO
351// write a "getIntegerWave" function??
352//
353//////  integer values
354// reads 32 bit integer
355Function V_getIntegerFromHDF5(fname,path)
356        String fname                            //full path+name
357        String path                             //path to the hdf5 location
358       
359        Variable val = V_getRealValueFromHDF5(fname,path)
360       
361        val = round(val)
362        return(val)
363End
364
365
366// read a single string
367// - fname passed in is the full path to the file on disk
368// - path is the path to the value in the HDF tree
369// - num is the number of characters in the VAX string
370// check to see if the value exists (It will be a wave)
371// -- if it does, return the value from the local folder
372// -- if not, read the file in, then return the value
373//
374// TODO -- string could be checked for length, but returned right or wrong
375//
376Function/S V_getStringFromHDF5(fname,path,num)
377        String fname,path
378        Variable num
379
380        String folderStr=""
381        Variable valExists=0
382       
383        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
384
385// check for a work folder first (note that "entry" is doubled)
386        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
387                Wave/Z/T tw = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
388                return(tw[0])
389        endif
390       
391        if(Exists(ksBaseDFPath+folderStr+":"+path))
392                valExists=1
393        endif
394       
395        if(!valExists)
396                //then read in the file
397                V_LoadHDF5_NoAtt(fname,"")
398        endif
399
400// this should exist now - if not, I need to see the error
401        Wave/T/Z tw = $(ksBaseDFPath+folderStr+":"+path)
402       
403        if(WaveExists(tw))
404       
405        //      if(strlen(tw[0]) != num)
406        //              Print "string is not the specified length"
407        //      endif
408               
409                return(tw[0])
410        else
411                return("The specified wave does not exist: " + path)
412        endif
413End
414
415
416
417///////////////////////////////
418
419//
420//Write Wave 'wav' to hdf5 file 'fname'
421//Based on code from ANSTO (N. Hauser. nha 8/1/09)
422//
423// TODO:
424// -- figure out if this will write in the native format of the
425//     wave as passed in, or if it will only write as DP.
426// -- do I need to write separate functions for real, integer, etc.?
427//     
428// -- change the /P=home to the user-defined data path (which may be home)             
429//
430Function V_WriteWaveToHDF(fname, groupName, varName, wav)
431        String fname, groupName, varName
432        Wave wav
433       
434        variable err=0, fileID,groupID
435        String cDF = getDataFolder(1), temp
436        String NXentry_name
437       
438        try     
439                HDF5OpenFile/P=home /Z fileID  as fname  //open file read-write
440                if(!fileID)
441                        err = 1
442                        abort "HDF5 file does not exist"
443                endif
444               
445                //get the NXentry node name
446                HDF5ListGroup /TYPE=1 fileID, "/"
447                //remove trailing ; from S_HDF5ListGroup
448               
449                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
450               
451                NXentry_name = S_HDF5ListGroup
452                NXentry_name = ReplaceString(";",NXentry_name,"")
453                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
454                        err = 1
455                        abort "More than one entry under the root node. Ambiguous"
456                endif
457                //concatenate NXentry node name and groupName   
458                // SRK - NOV2015 - dropped this and require the full group name passed in
459//              groupName = "/" + NXentry_name + groupName
460                Print "groupName = ",groupName
461                HDF5OpenGroup /Z fileID , groupName, groupID
462
463                if(!groupID)
464                // don't create the group it the name isn't right -- throw up an error
465                        //HDF5CreateGroup /Z fileID, groupName, groupID
466                        err = 1
467                        HDF5CloseFile /Z fileID
468                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
469                        return(err)
470                else
471                        // get attributes and save them
472                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
473                        //Wave attributes = S_HDF5ListAttributes
474                endif
475       
476                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
477                if (V_flag != 0)
478                        err = 1
479                        abort "Cannot save wave to HDF5 dataset " + varName
480                endif   
481               
482               
483                //attributes - something could be added here as optional parameters and flagged
484//              String attributes = "units"
485//              Make/O/T/N=1 tmp
486//              tmp[0] = "dimensionless"
487//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
488//              if (V_flag != 0)
489//                      err = 1
490//                      abort "Cannot save attributes to HDF5 dataset"
491//              endif   
492        catch
493
494        endtry
495
496// it is not necessary to close the group here. HDF5CloseFile will close the group as well     
497        if(groupID)
498                HDF5CloseGroup /Z groupID
499        endif
500       
501        if(fileID)
502                HDF5CloseFile /Z fileID
503        endif
504
505        setDataFolder $cDF
506        return err
507end
508
509//Write Wave 'wav' to hdf5 file 'fname'
510//Based on code from ANSTO (N. Hauser. nha 8/1/09)
511//
512// TODO
513//
514// -- change the /P=home to the user-defined data path (which may be home)             
515//
516Function V_WriteTextWaveToHDF(fname, groupName, varName, wav)
517        String fname, groupName, varName
518        Wave/T wav
519       
520        variable err=0, fileID,groupID
521        String cDF = getDataFolder(1), temp
522        String NXentry_name
523       
524        try     
525                HDF5OpenFile/P=home /Z fileID  as fname  //open file read-write
526                if(!fileID)
527                        err = 1
528                        abort "HDF5 file does not exist"
529                endif
530               
531                //get the NXentry node name
532                HDF5ListGroup /TYPE=1 fileID, "/"
533                //remove trailing ; from S_HDF5ListGroup
534               
535                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
536               
537                NXentry_name = S_HDF5ListGroup
538                NXentry_name = ReplaceString(";",NXentry_name,"")
539                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
540                        err = 1
541                        abort "More than one entry under the root node. Ambiguous"
542                endif
543
544                //concatenate NXentry node name and groupName
545                // SRK - NOV2015 - dropped this and require the full group name passed in
546//              groupName = "/" + NXentry_name + groupName
547                Print "groupName = ",groupName
548
549                HDF5OpenGroup /Z fileID , groupName, groupID
550
551                if(!groupID)
552                // don't create the group it the name isn't right -- throw up an error
553                        //HDF5CreateGroup /Z fileID, groupName, groupID
554                        err = 1
555                        HDF5CloseFile /Z fileID
556                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
557                        return(err)
558                else
559                        // get attributes and save them
560                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
561                        //Wave attributes = S_HDF5ListAttributes
562                endif
563       
564                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
565                if (V_flag != 0)
566                        err = 1
567                        abort "Cannot save wave to HDF5 dataset " + varName
568                endif   
569               
570               
571                //attributes - something could be added here as optional parameters and flagged
572//              String attributes = "units"
573//              Make/O/T/N=1 tmp
574//              tmp[0] = "dimensionless"
575//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
576//              if (V_flag != 0)
577//                      err = 1
578//                      abort "Cannot save attributes to HDF5 dataset"
579//              endif   
580        catch
581
582        endtry
583       
584        if(groupID)
585                HDF5CloseGroup /Z groupID
586        endif
587       
588        if(fileID)
589                HDF5CloseFile /Z fileID
590        endif
591
592        setDataFolder $cDF
593        return err
594end
595
596//////////////////////////////
597//////////////////////////////
598//////////////////////////////
599
600Function V_KillNamedDataFolder(fname)
601        String fname
602       
603        Variable err=0
604       
605        String folderStr = V_GetFileNameFromPathNoSemi(fname)
606        folderStr = V_RemoveDotExtension(folderStr)
607       
608        KillDataFolder/Z $(ksBaseDFPath+folderStr)
609        err = V_flag
610       
611        return(err)
612end
613
614//given a filename of a SANS data filename of the form
615// name.anything
616//returns the name as a string without the ".fbdfasga" extension
617//
618// returns the input string if a"." can't be found (maybe it wasn't there"
619Function/S V_RemoveDotExtension(item)
620        String item
621        String invalid = item   //
622        Variable num=-1
623       
624        //find the "dot"
625        String runStr=""
626        Variable pos = strsearch(item,".",0)
627        if(pos == -1)
628                //"dot" not found
629                return (invalid)
630        else
631                //found, get all of the characters preceeding it
632                runStr = item[0,pos-1]
633                return (runStr)
634        Endif
635End
636
637//returns a string containing filename (WITHOUT the ;vers)
638//the input string is a full path to the file (Mac-style, still works on Win in IGOR)
639//with the folders separated by colons
640//
641// called by MaskUtils.ipf, ProtocolAsPanel.ipf, WriteQIS.ipf
642//
643Function/S V_GetFileNameFromPathNoSemi(fullPath)
644        String fullPath
645       
646        Variable offset1,offset2
647        String filename=""
648        //String PartialPath
649        offset1 = 0
650        do
651                offset2 = StrSearch(fullPath, ":", offset1)
652                if (offset2 == -1)                              // no more colons ?
653                        fileName = FullPath[offset1,strlen(FullPath) ]
654                        //PartialPath = FullPath[0, offset1-1]
655                        break
656                endif
657                offset1 = offset2+1
658        while (1)
659       
660        //remove version number from name, if it's there - format should be: filename;N
661        filename =  StringFromList(0,filename,";")              //returns null if error
662       
663        Return filename
664End
Note: See TracBrowser for help on using the repository browser.