source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/V_HDF5_RW_Utils.ipf @ 980

Last change on this file since 980 was 980, checked in by srkline, 7 years ago

more minor changes

File size: 18.3 KB
Line 
1#pragma rtGlobals=3             // Use modern global access method and strict wave access.
2
3//
4// The base functions for R/W from HDF5 files.
5// All of the specific "get" and "write" functions call these base functions which are responsible
6// for all of the open/close mechanics.
7//
8// These VSANS file-specific functions are in:
9// V_HDF5_Read.ipf
10//               and
11// V_HDF5_Write.ipf
12//
13
14// data is read into:
15//      NewDataFolder/O/S root:Packages:NIST:VSANS:RawVSANS
16// so that the data folders of the raw data won't be lying around on the top level, looking like
17// 1D data sets and interfering.
18
19
20// the base data folder path where the raw data is loaded
21Strconstant ksBaseDFPath = "root:Packages:NIST:VSANS:RawVSANS:"
22
23
24// passing null file string presents a dialog
25Proc LoadFakeDIVData()
26        V_LoadHDF5Data("","DIV")
27End
28
29// passing null file string presents a dialog
30Proc Read_HDF5_Raw_No_Attributes()
31        V_LoadHDF5Data("","RAW")
32End
33
34
35// TODO:
36//  x- move the initialization of the raw data folder to be in the as-yet unwritten initialization routine for
37// reduction. be sure that it's duplicated in the VCALC initialization too.
38// -- as needed, get rid of the FAKE redimension of the data from 3D->2D and from 128x128 to something else for VSANS
39//    This is a fake since I don't have anything close to correct fake data yet. (1/29/16)
40//
41// TODO: -- is there an extra "entry" heading? Am I adding this by mistake by setting base_name="entry" for RAW data?
42//
43Function V_LoadHDF5Data(file,folder)
44        String file,folder
45
46        String base_name
47        SetDataFolder $("root:Packages:NIST:VSANS:"+folder)
48//      SetDataFolder root:
49        if(cmpstr(folder,"RAW")==0)
50                base_name="entry"
51        else
52                base_name="entry"               //TODO -- remove this / change behavior in V_LoadHDF5_NoAtt()
53        endif
54       
55        Variable err= V_LoadHDF5_NoAtt(file,base_name)  // reads into current folder
56       
57        // if RAW data, then generate the errors and linear data copy
58        // do this 9x
59        string tmpStr = "root:Packages:NIST:VSANS:RAW:entry:entry:instrument:"
60        if(cmpstr(folder,"RAW")==0)
61                V_MakeDataError(tmpStr+"detector_B")
62                V_MakeDataError(tmpStr+"detector_MB")
63                V_MakeDataError(tmpStr+"detector_MT")
64                V_MakeDataError(tmpStr+"detector_ML")
65                V_MakeDataError(tmpStr+"detector_MR")
66                V_MakeDataError(tmpStr+"detector_FB")
67                V_MakeDataError(tmpStr+"detector_FT")
68                V_MakeDataError(tmpStr+"detector_FL")
69                V_MakeDataError(tmpStr+"detector_FR")
70
71// TODO -- once I get "real" data, get rid of this call to force the data to be proper dimensions.
72                V_RedimFakeData()
73               
74/// END FAKE DATA CORRECTIONS           
75               
76        endif
77       
78        SetDataFolder root:
79        return(err)
80End
81
82//
83// TODO -- this is all FAKED since all the data arrays are (1,128,128)
84// I'm intentionally using the wrong number of pixels so I'm more likely to go back and get rid of this later.
85//
86// the SetScale parts may be useful later.
87//
88Function V_RedimFakeData()
89       
90                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_B
91                Wave det_B=data
92                Redimension/N=(320,320)/E=1 det_B
93               
94                Variable ctr=20,npix=128
95                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MT
96                Wave det_MT=data
97                Redimension/N=(npix,48)/E=1 det_MT             
98                SetScale/I x -npix/2,npix/2,"",det_MT
99                SetScale/I y ctr,ctr+48,"",det_MT
100//              det_mt[][20] = 50
101        det_MT *= 10
102        det_MT += 2
103
104                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MB
105                Wave det_MB=data
106                Redimension/N=(npix,48)/E=1 det_MB             
107                SetScale/I x -npix/2,npix/2,"",det_MB
108                SetScale/I y -ctr,-ctr-48,"",det_MB
109        det_MB *= 5
110        det_MB += 2
111               
112                ctr=30
113                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_ML
114                Wave det_ML=data
115                Redimension/N=(48,npix)/E=1 det_ML             
116                SetScale/I x -ctr-48,-ctr,"",det_ML
117                SetScale/I y -npix/2,npix/2,"",det_ML
118        det_ML *= 2
119        det_ML += 2
120       
121                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MR
122                Wave det_MR=data
123                Redimension/N=(48,npix)/E=1 det_MR             
124                SetScale/I x ctr,ctr+48,"",det_MR
125                SetScale/I y -npix/2,npix/2,"",det_MR
126        det_MR +=2
127               
128                ctr=30
129                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FT
130                Wave det_FT=data
131                Redimension/N=(npix,48)/E=1 det_FT             
132                SetScale/I x -npix/2,npix/2,"",det_FT
133                SetScale/I y ctr,ctr+48,"",det_FT
134
135                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FB
136                Wave det_FB=data
137                Redimension/N=(npix,48)/E=1 det_FB             
138                SetScale/I x -npix/2,npix/2,"",det_FB
139                SetScale/I y -ctr,-ctr-48,"",det_FB
140               
141                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FL
142                Wave det_FL=data
143                Redimension/N=(48,npix)/E=1 det_FL             
144                SetScale/I x -ctr-48,-ctr,"",det_FL
145                SetScale/I y -npix/2,npix/2,"",det_FL
146               
147                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FR
148                Wave det_FR=data
149                Redimension/N=(48,npix)/E=1 det_FR             
150                SetScale/I x ctr,ctr+48,"",det_FR
151                SetScale/I y -npix/2,npix/2,"",det_FR
152
153
154        return(0)
155End
156
157
158// This loads for speed, since loading the attributes takes a LOT of time.
159//
160// this will load in the whole HDF file all at once.
161// Attributes are NOT loaded at all.
162//
163// TODO: remove the P=home restriction top make this more generic
164// -- get rid of bits leftover here that I don't need
165// -- be sure I'm using all of the correct flags in the HDF5LoadGroup operation
166// -- settle on how the base_name is to be used. "entry" for the RAW, fileName for the "rawVSANS"?
167//
168// passing in "" for base_name will take the name from the file name as selected
169//
170Function V_LoadHDF5_NoAtt(fileName,base_name)
171        String fileName, base_name
172       
173//      if ( ParamIsDefault(hdf5Path) )
174//              hdf5Path = "/"
175//      endif
176
177        String hdf5path = "/"           //always read from the top
178        String status = ""
179
180        Variable fileID = 0
181        HDF5OpenFile/R/P=home/Z fileID as fileName              //read file from home directory?
182//      HDF5OpenFile/R/P=catPathName/Z fileID as fileName
183        if (V_Flag != 0)
184                return 0
185        endif
186
187        String/G root:file_path = S_path
188        String/G root:file_name = S_FileName
189       
190        if ( fileID == 0 )
191                Print fileName + ": could not open as HDF5 file"
192                return (0)
193        endif
194       
195//s_tic()               //fast
196       
197        SVAR tmpStr=root:file_name
198        fileName=tmpStr         //SRK - in case the file was chosen from a dialog, I'll need access to the name later
199       
200        //   read the data (too bad that HDF5LoadGroup does not read the attributes)
201        if(cmpstr(base_name,"") == 0)
202                base_name = StringFromList(0,FileName,".")
203        endif
204        //base_name = "entry"
205       
206        HDF5LoadGroup/Z/L=7/O/R/T=$base_name  :, fileID, hdf5Path               //      recursive
207        if ( V_Flag != 0 )
208                Print fileName + ": could not open as HDF5 file"
209                setdatafolder root:
210                return (0)
211        endif
212
213        HDF5CloseFile fileID
214       
215//s_toc()
216        return(0)
217end     
218
219
220// read a single real value
221// - fname passed in is the full path to the file on disk
222// - path is the path to the value in the HDF tree
223//
224// check to see if the value exists (It will be a wave)
225// -- if it does, return the value from the local folder
226// -- if not, read the file in, then return the value
227//
228// TODO:
229// currently, the work folders have the following path - so passing in "RAW" as fname
230// will take some re-configuring.
231//  root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FL:distance
232// -- be sure this read from work folders is not broken in the future, and is passed to ALL of the
233//    top-level R/W routines. (Write is necessary ONLY for SIM data files. Patch is direct to disk.)
234Function V_getRealValueFromHDF5(fname,path)
235        String fname,path
236
237        String folderStr=""
238        Variable valExists=0
239       
240        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
241
242// check for a work folder first (note that "entry" is doubled)
243        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
244                Wave/Z w = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
245                return(w[0])
246        endif
247       
248        if(Exists(ksBaseDFPath+folderStr+":"+path))
249                valExists=1
250        endif
251       
252        if(!valExists)
253                //then read in the file
254                V_LoadHDF5_NoAtt(fname,"")
255        endif
256
257// this should exist now - if not, I need to see the error
258        Wave/Z w = $(ksBaseDFPath+folderStr+":"+path)
259       
260        if(WaveExists(w))
261                return(w[0])
262        else
263                return(-999999)
264        endif   
265End
266
267// Returns a wave reference, not just a single value
268// ---then you pick what you need from the wave
269//
270// - fname passed in is the full path to the file on disk
271// - path is the path to the value in the HDF tree
272//
273// check to see if the value exists (It will be a wave)
274// -- if it does, return the value from the local folder
275// -- if not, read the file in, then return the value
276//
277Function/WAVE V_getRealWaveFromHDF5(fname,path)
278        String fname,path
279
280        String folderStr=""
281        Variable valExists=0
282       
283        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
284
285// check for a work folder first (note that "entry" is doubled)
286        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
287                Wave wOut = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
288                return wOut
289        endif
290               
291        if(Exists(ksBaseDFPath+folderStr+":"+path))
292                valExists=1
293        endif
294       
295        if(!valExists)
296                //then read in the file
297                V_LoadHDF5_NoAtt(fname,"")
298        endif
299
300// this should exist now - if not, I need to see the error
301        Wave wOut = $(ksBaseDFPath+folderStr+":"+path)
302       
303        return wOut
304       
305End
306
307// Returns a wave reference, not just a single value
308// ---then you pick what you need from the wave
309//
310// - fname passed in is the full path to the file on disk
311// - path is the path to the value in the HDF tree
312//
313// check to see if the value exists (It will be a wave)
314// -- if it does, return the value from the local folder
315// -- if not, read the file in, then return the value
316//
317Function/WAVE V_getTextWaveFromHDF5(fname,path)
318        String fname,path
319
320        String folderStr=""
321        Variable valExists=0
322       
323        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
324
325// check for a work folder first (note that "entry" is doubled)
326        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
327                Wave/T wOut = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
328                return wOut
329        endif
330       
331        if(Exists(ksBaseDFPath+folderStr+":"+path))
332                valExists=1
333        endif
334       
335        if(!valExists)
336                //then read in the file
337                V_LoadHDF5_NoAtt(fname,"")
338        endif
339
340// this should exist now - if not, I need to see the error
341        Wave/T wOut = $(ksBaseDFPath+folderStr+":"+path)
342       
343        return wOut
344       
345End
346
347
348//
349//   TODO
350// depricated? in HDF5 - store all of the values as real?
351// Igor sees no difference in real and integer variables (waves are different)
352// BUT-- Igor 7 will have integer variables
353//
354// truncate to integer before returning??
355//
356//  TODO
357// write a "getIntegerWave" function??
358//
359//////  integer values
360// reads 32 bit integer
361Function V_getIntegerFromHDF5(fname,path)
362        String fname                            //full path+name
363        String path                             //path to the hdf5 location
364       
365        Variable val = V_getRealValueFromHDF5(fname,path)
366       
367        val = round(val)
368        return(val)
369End
370
371
372// read a single string
373// - fname passed in is the full path to the file on disk
374// - path is the path to the value in the HDF tree
375// - num is the number of characters in the VAX string
376// check to see if the value exists (It will be a wave)
377// -- if it does, return the value from the local folder
378// -- if not, read the file in, then return the value
379//
380// TODO -- string could be checked for length, but returned right or wrong
381//
382Function/S V_getStringFromHDF5(fname,path,num)
383        String fname,path
384        Variable num
385
386        String folderStr=""
387        Variable valExists=0
388       
389        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
390
391// check for a work folder first (note that "entry" is doubled)
392        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
393                Wave/Z/T tw = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
394                return(tw[0])
395        endif
396       
397        if(Exists(ksBaseDFPath+folderStr+":"+path))
398                valExists=1
399        endif
400       
401        if(!valExists)
402                //then read in the file
403                V_LoadHDF5_NoAtt(fname,"")
404        endif
405
406// this should exist now - if not, I need to see the error
407        Wave/T/Z tw = $(ksBaseDFPath+folderStr+":"+path)
408       
409        if(WaveExists(tw))
410       
411        //      if(strlen(tw[0]) != num)
412        //              Print "string is not the specified length"
413        //      endif
414               
415                return(tw[0])
416        else
417                return("The specified wave does not exist: " + path)
418        endif
419End
420
421
422
423///////////////////////////////
424
425//
426//Write Wave 'wav' to hdf5 file 'fname'
427//Based on code from ANSTO (N. Hauser. nha 8/1/09)
428//
429// TODO:
430// -- figure out if this will write in the native format of the
431//     wave as passed in, or if it will only write as DP.
432// -- do I need to write separate functions for real, integer, etc.?
433//     
434// -- change the /P=home to the user-defined data path (which may be home)             
435//
436Function V_WriteWaveToHDF(fname, groupName, varName, wav)
437        String fname, groupName, varName
438        Wave wav
439       
440        variable err=0, fileID,groupID
441        String cDF = getDataFolder(1), temp
442        String NXentry_name
443       
444        try     
445                HDF5OpenFile/P=home /Z fileID  as fname  //open file read-write
446                if(!fileID)
447                        err = 1
448                        abort "HDF5 file does not exist"
449                endif
450               
451                //get the NXentry node name
452                HDF5ListGroup /TYPE=1 fileID, "/"
453                //remove trailing ; from S_HDF5ListGroup
454               
455                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
456               
457                NXentry_name = S_HDF5ListGroup
458                NXentry_name = ReplaceString(";",NXentry_name,"")
459                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
460                        err = 1
461                        abort "More than one entry under the root node. Ambiguous"
462                endif
463                //concatenate NXentry node name and groupName   
464                // SRK - NOV2015 - dropped this and require the full group name passed in
465//              groupName = "/" + NXentry_name + groupName
466                Print "groupName = ",groupName
467                HDF5OpenGroup /Z fileID , groupName, groupID
468
469                if(!groupID)
470                // don't create the group it the name isn't right -- throw up an error
471                        //HDF5CreateGroup /Z fileID, groupName, groupID
472                        err = 1
473                        HDF5CloseFile /Z fileID
474                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
475                        return(err)
476                else
477                        // get attributes and save them
478                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
479                        //Wave attributes = S_HDF5ListAttributes
480                endif
481       
482                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
483                if (V_flag != 0)
484                        err = 1
485                        abort "Cannot save wave to HDF5 dataset " + varName
486                endif   
487               
488               
489                //attributes - something could be added here as optional parameters and flagged
490//              String attributes = "units"
491//              Make/O/T/N=1 tmp
492//              tmp[0] = "dimensionless"
493//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
494//              if (V_flag != 0)
495//                      err = 1
496//                      abort "Cannot save attributes to HDF5 dataset"
497//              endif   
498        catch
499
500        endtry
501
502// it is not necessary to close the group here. HDF5CloseFile will close the group as well     
503        if(groupID)
504                HDF5CloseGroup /Z groupID
505        endif
506       
507        if(fileID)
508                HDF5CloseFile /Z fileID
509        endif
510
511        setDataFolder $cDF
512        return err
513end
514
515//Write Wave 'wav' to hdf5 file 'fname'
516//Based on code from ANSTO (N. Hauser. nha 8/1/09)
517//
518// TODO
519//
520// -- change the /P=home to the user-defined data path (which may be home)             
521//
522Function V_WriteTextWaveToHDF(fname, groupName, varName, wav)
523        String fname, groupName, varName
524        Wave/T wav
525       
526        variable err=0, fileID,groupID
527        String cDF = getDataFolder(1), temp
528        String NXentry_name
529       
530        try     
531                HDF5OpenFile/P=home /Z fileID  as fname  //open file read-write
532                if(!fileID)
533                        err = 1
534                        abort "HDF5 file does not exist"
535                endif
536               
537                //get the NXentry node name
538                HDF5ListGroup /TYPE=1 fileID, "/"
539                //remove trailing ; from S_HDF5ListGroup
540               
541                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
542               
543                NXentry_name = S_HDF5ListGroup
544                NXentry_name = ReplaceString(";",NXentry_name,"")
545                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
546                        err = 1
547                        abort "More than one entry under the root node. Ambiguous"
548                endif
549
550                //concatenate NXentry node name and groupName
551                // SRK - NOV2015 - dropped this and require the full group name passed in
552//              groupName = "/" + NXentry_name + groupName
553                Print "groupName = ",groupName
554
555                HDF5OpenGroup /Z fileID , groupName, groupID
556
557                if(!groupID)
558                // don't create the group it the name isn't right -- throw up an error
559                        //HDF5CreateGroup /Z fileID, groupName, groupID
560                        err = 1
561                        HDF5CloseFile /Z fileID
562                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
563                        return(err)
564                else
565                        // get attributes and save them
566                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
567                        //Wave attributes = S_HDF5ListAttributes
568                endif
569       
570                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
571                if (V_flag != 0)
572                        err = 1
573                        abort "Cannot save wave to HDF5 dataset " + varName
574                endif   
575               
576               
577                //attributes - something could be added here as optional parameters and flagged
578//              String attributes = "units"
579//              Make/O/T/N=1 tmp
580//              tmp[0] = "dimensionless"
581//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
582//              if (V_flag != 0)
583//                      err = 1
584//                      abort "Cannot save attributes to HDF5 dataset"
585//              endif   
586        catch
587
588        endtry
589       
590        if(groupID)
591                HDF5CloseGroup /Z groupID
592        endif
593       
594        if(fileID)
595                HDF5CloseFile /Z fileID
596        endif
597
598        setDataFolder $cDF
599        return err
600end
601
602//////////////////////////////
603//////////////////////////////
604//////////////////////////////
605
606Function V_KillNamedDataFolder(fname)
607        String fname
608       
609        Variable err=0
610       
611        String folderStr = V_GetFileNameFromPathNoSemi(fname)
612        folderStr = V_RemoveDotExtension(folderStr)
613       
614        KillDataFolder/Z $(ksBaseDFPath+folderStr)
615        err = V_flag
616       
617        return(err)
618end
619
620//given a filename of a SANS data filename of the form
621// name.anything
622//returns the name as a string without the ".fbdfasga" extension
623//
624// returns the input string if a"." can't be found (maybe it wasn't there"
625Function/S V_RemoveDotExtension(item)
626        String item
627        String invalid = item   //
628        Variable num=-1
629       
630        //find the "dot"
631        String runStr=""
632        Variable pos = strsearch(item,".",0)
633        if(pos == -1)
634                //"dot" not found
635                return (invalid)
636        else
637                //found, get all of the characters preceeding it
638                runStr = item[0,pos-1]
639                return (runStr)
640        Endif
641End
642
643//returns a string containing filename (WITHOUT the ;vers)
644//the input string is a full path to the file (Mac-style, still works on Win in IGOR)
645//with the folders separated by colons
646//
647// called by MaskUtils.ipf, ProtocolAsPanel.ipf, WriteQIS.ipf
648//
649Function/S V_GetFileNameFromPathNoSemi(fullPath)
650        String fullPath
651       
652        Variable offset1,offset2
653        String filename=""
654        //String PartialPath
655        offset1 = 0
656        do
657                offset2 = StrSearch(fullPath, ":", offset1)
658                if (offset2 == -1)                              // no more colons ?
659                        fileName = FullPath[offset1,strlen(FullPath) ]
660                        //PartialPath = FullPath[0, offset1-1]
661                        break
662                endif
663                offset1 = offset2+1
664        while (1)
665       
666        //remove version number from name, if it's there - format should be: filename;N
667        filename =  StringFromList(0,filename,";")              //returns null if error
668       
669        Return filename
670End
Note: See TracBrowser for help on using the repository browser.