source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/V_HDF5_RW_Utils.ipf @ 978

Last change on this file since 978 was 978, checked in by srkline, 7 years ago

more additions to the raw data display panel

File size: 17.9 KB
Line 
1#pragma rtGlobals=3             // Use modern global access method and strict wave access.
2
3//
4// The base functions for R/W from HDF5 files.
5// All of the specific "get" and "write" functions call these base functions which are responsible
6// for all of the open/close mechanics.
7//
8// These VSANS file-specific functions are in:
9// V_HDF5_Read.ipf
10//               and
11// V_HDF5_Write.ipf
12//
13
14// data is read into:
15//      NewDataFolder/O/S root:Packages:NIST:VSANS:RawVSANS
16// so that the data folders of the raw data won't be lying around on the top level, looking like
17// 1D data sets and interfering.
18
19
20// the base data folder path where the raw data is loaded
21Strconstant ksBaseDFPath = "root:Packages:NIST:VSANS:RawVSANS:"
22
23
24
25// passing null file string presents a dialog
26Proc Read_HDF5_Raw_No_Attributes()
27        V_LoadHDF5Data("","RAW")
28End
29
30
31// TODO:
32//  x- move the initializtion of the raw data folder to be in the as-yet unwritten initialization routine for
33// reduction. be sure that it's duplicated in the VCALC initialization too.
34// -- as needed, get rid of the FAKE redimension of the data from 3D->2D and from 128x128 to something else for VSANS
35//    This is a fake since I don't have anything close to correct fake data yet. (1/29/16)
36//
37Function V_LoadHDF5Data(file,folder)
38        String file,folder
39
40        String base_name
41        SetDataFolder $("root:Packages:NIST:VSANS:"+folder)
42//      SetDataFolder root:
43        if(cmpstr(folder,"RAW")==0)
44                base_name="entry"
45        else
46                base_name=""
47        endif
48       
49        Variable err= V_LoadHDF5_NoAtt(file,base_name)  // reads into current folder
50       
51        // if RAW data, then generate the errors and linear data copy
52        // do this 9x
53        string tmpStr = "root:Packages:NIST:VSANS:RAW:entry:entry:instrument:"
54        if(cmpstr(folder,"RAW")==0)
55                V_MakeDataError(tmpStr+"detector_B")
56                V_MakeDataError(tmpStr+"detector_MB")
57                V_MakeDataError(tmpStr+"detector_MT")
58                V_MakeDataError(tmpStr+"detector_ML")
59                V_MakeDataError(tmpStr+"detector_MR")
60                V_MakeDataError(tmpStr+"detector_FB")
61                V_MakeDataError(tmpStr+"detector_FT")
62                V_MakeDataError(tmpStr+"detector_FL")
63                V_MakeDataError(tmpStr+"detector_FR")
64
65// TODO -- once I get "real" data, get rid of this call to force the data to be proper dimensions.
66                V_RedimFakeData()
67               
68/// END FAKE DATA CORRECTIONS           
69               
70        endif
71       
72        SetDataFolder root:
73        return(err)
74End
75
76//
77// TODO -- this is all FAKED since all the data arrays are (1,128,128)
78// I'm intentionally using the wrong number of pixels so I'm more likely to go back and get rid of this later.
79//
80// the SetScale parts may be useful later.
81//
82Function V_RedimFakeData()
83       
84                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_B
85                Wave det_B=data
86                Redimension/N=(300,300)/E=1 det_B
87               
88                Variable ctr=20,npix=100
89                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MT
90                Wave det_MT=data
91                Redimension/N=(npix,48)/E=1 det_MT             
92                SetScale/I x -npix/2,npix/2,"",det_MT
93                SetScale/I y ctr,ctr+48,"",det_MT
94//              det_mt[][20] = 50
95        det_MT *= 10
96
97                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MB
98                Wave det_MB=data
99                Redimension/N=(npix,48)/E=1 det_MB             
100                SetScale/I x -npix/2,npix/2,"",det_MB
101                SetScale/I y -ctr,-ctr-48,"",det_MB
102        det_MB *= 5
103               
104                ctr=30
105                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_ML
106                Wave det_ML=data
107                Redimension/N=(48,npix)/E=1 det_ML             
108                SetScale/I x -ctr-48,-ctr,"",det_ML
109                SetScale/I y -npix/2,npix/2,"",det_ML
110        det_ML *= 2
111               
112                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MR
113                Wave det_MR=data
114                Redimension/N=(48,npix)/E=1 det_MR             
115                SetScale/I x ctr,ctr+48,"",det_MR
116                SetScale/I y -npix/2,npix/2,"",det_MR
117               
118                ctr=30
119                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FT
120                Wave det_FT=data
121                Redimension/N=(npix,48)/E=1 det_FT             
122                SetScale/I x -npix/2,npix/2,"",det_FT
123                SetScale/I y ctr,ctr+48,"",det_FT
124
125                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FB
126                Wave det_FB=data
127                Redimension/N=(npix,48)/E=1 det_FB             
128                SetScale/I x -npix/2,npix/2,"",det_FB
129                SetScale/I y -ctr,-ctr-48,"",det_FB
130               
131                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FL
132                Wave det_FL=data
133                Redimension/N=(48,npix)/E=1 det_FL             
134                SetScale/I x -ctr-48,-ctr,"",det_FL
135                SetScale/I y -npix/2,npix/2,"",det_FL
136               
137                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FR
138                Wave det_FR=data
139                Redimension/N=(48,npix)/E=1 det_FR             
140                SetScale/I x ctr,ctr+48,"",det_FR
141                SetScale/I y -npix/2,npix/2,"",det_FR
142
143
144        return(0)
145End
146
147
148// This loads for speed, since loading the attributes takes a LOT of time.
149//
150// this will load in the whole HDF file all at once.
151// Attributes are NOT loaded at all.
152//
153// TODO: remove the P=home restriction top make this more generic
154// -- get rid of bits leftover here that I don't need
155// -- be sure I'm using all of the correct flags in the HDF5LoadGroup operation
156// -- settle on how the base_name is to be used. "entry" for the RAW, fileName for the "rawVSANS"?
157//
158// passing in "" for base_name will take the name from the file name as selected
159//
160Function V_LoadHDF5_NoAtt(fileName,base_name)
161        String fileName, base_name
162       
163//      if ( ParamIsDefault(hdf5Path) )
164//              hdf5Path = "/"
165//      endif
166
167        String hdf5path = "/"           //always read from the top
168        String status = ""
169
170        Variable fileID = 0
171        HDF5OpenFile/R/P=home/Z fileID as fileName              //read file from home directory?
172//      HDF5OpenFile/R/P=catPathName/Z fileID as fileName
173        if (V_Flag != 0)
174                return 0
175        endif
176
177        String/G root:file_path = S_path
178        String/G root:file_name = S_FileName
179       
180        if ( fileID == 0 )
181                Print fileName + ": could not open as HDF5 file"
182                return (0)
183        endif
184       
185//s_tic()               //fast
186       
187        SVAR tmpStr=root:file_name
188        fileName=tmpStr         //SRK - in case the file was chosen from a dialog, I'll need access to the name later
189       
190        //   read the data (too bad that HDF5LoadGroup does not read the attributes)
191        if(cmpstr(base_name,"") == 0)
192                base_name = StringFromList(0,FileName,".")
193        endif
194        //base_name = "entry"
195       
196        HDF5LoadGroup/Z/L=7/O/R/T=$base_name  :, fileID, hdf5Path               //      recursive
197        if ( V_Flag != 0 )
198                Print fileName + ": could not open as HDF5 file"
199                setdatafolder root:
200                return (0)
201        endif
202
203        HDF5CloseFile fileID
204       
205//s_toc()
206        return(0)
207end     
208
209
210// read a single real value
211// - fname passed in is the full path to the file on disk
212// - path is the path to the value in the HDF tree
213//
214// check to see if the value exists (It will be a wave)
215// -- if it does, return the value from the local folder
216// -- if not, read the file in, then return the value
217//
218// TODO:
219// currently, the work folders have the following path - so passing in "RAW" as fname
220// will take some re-configuring.
221//  root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FL:distance
222// -- be sure this read from work folders is not broken in the future, and is passed to ALL of the
223//    top-level R/W routines. (Write is necessary ONLY for SIM data files. Patch is direct to disk.)
224Function V_getRealValueFromHDF5(fname,path)
225        String fname,path
226
227        String folderStr=""
228        Variable valExists=0
229       
230        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
231
232// check for a work folder first (note that "entry" is doubled)
233        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
234                Wave/Z w = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
235                return(w[0])
236        endif
237       
238        if(Exists(ksBaseDFPath+folderStr+":"+path))
239                valExists=1
240        endif
241       
242        if(!valExists)
243                //then read in the file
244                V_LoadHDF5_NoAtt(fname,"")
245        endif
246
247// this should exist now - if not, I need to see the error
248        Wave/Z w = $(ksBaseDFPath+folderStr+":"+path)
249       
250        if(WaveExists(w))
251                return(w[0])
252        else
253                return(-999999)
254        endif   
255End
256
257// Returns a wave reference, not just a single value
258// ---then you pick what you need from the wave
259//
260// - fname passed in is the full path to the file on disk
261// - path is the path to the value in the HDF tree
262//
263// check to see if the value exists (It will be a wave)
264// -- if it does, return the value from the local folder
265// -- if not, read the file in, then return the value
266//
267Function/WAVE V_getRealWaveFromHDF5(fname,path)
268        String fname,path
269
270        String folderStr=""
271        Variable valExists=0
272       
273        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
274
275// check for a work folder first (note that "entry" is doubled)
276        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
277                Wave wOut = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
278                return wOut
279        endif
280               
281        if(Exists(ksBaseDFPath+folderStr+":"+path))
282                valExists=1
283        endif
284       
285        if(!valExists)
286                //then read in the file
287                V_LoadHDF5_NoAtt(fname,"")
288        endif
289
290// this should exist now - if not, I need to see the error
291        Wave wOut = $(ksBaseDFPath+folderStr+":"+path)
292       
293        return wOut
294       
295End
296
297// Returns a wave reference, not just a single value
298// ---then you pick what you need from the wave
299//
300// - fname passed in is the full path to the file on disk
301// - path is the path to the value in the HDF tree
302//
303// check to see if the value exists (It will be a wave)
304// -- if it does, return the value from the local folder
305// -- if not, read the file in, then return the value
306//
307Function/WAVE V_getTextWaveFromHDF5(fname,path)
308        String fname,path
309
310        String folderStr=""
311        Variable valExists=0
312       
313        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
314
315// check for a work folder first (note that "entry" is doubled)
316        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
317                Wave/T wOut = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
318                return wOut
319        endif
320       
321        if(Exists(ksBaseDFPath+folderStr+":"+path))
322                valExists=1
323        endif
324       
325        if(!valExists)
326                //then read in the file
327                V_LoadHDF5_NoAtt(fname,"")
328        endif
329
330// this should exist now - if not, I need to see the error
331        Wave/T wOut = $(ksBaseDFPath+folderStr+":"+path)
332       
333        return wOut
334       
335End
336
337
338//
339//   TODO
340// depricated? in HDF5 - store all of the values as real?
341// Igor sees no difference in real and integer variables (waves are different)
342// BUT-- Igor 7 will have integer variables
343//
344// truncate to integer before returning??
345//
346//  TODO
347// write a "getIntegerWave" function??
348//
349//////  integer values
350// reads 32 bit integer
351Function V_getIntegerFromHDF5(fname,path)
352        String fname                            //full path+name
353        String path                             //path to the hdf5 location
354       
355        Variable val = V_getRealValueFromHDF5(fname,path)
356       
357        val = round(val)
358        return(val)
359End
360
361
362// read a single string
363// - fname passed in is the full path to the file on disk
364// - path is the path to the value in the HDF tree
365// - num is the number of characters in the VAX string
366// check to see if the value exists (It will be a wave)
367// -- if it does, return the value from the local folder
368// -- if not, read the file in, then return the value
369//
370// TODO -- string could be checked for length, but returned right or wrong
371//
372Function/S V_getStringFromHDF5(fname,path,num)
373        String fname,path
374        Variable num
375
376        String folderStr=""
377        Variable valExists=0
378       
379        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
380
381// check for a work folder first (note that "entry" is doubled)
382        if(Exists("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path))
383                Wave/Z/T tw = $("root:Packages:NIST:VSANS:"+folderStr+":entry:"+path)
384                return(tw[0])
385        endif
386       
387        if(Exists(ksBaseDFPath+folderStr+":"+path))
388                valExists=1
389        endif
390       
391        if(!valExists)
392                //then read in the file
393                V_LoadHDF5_NoAtt(fname,"")
394        endif
395
396// this should exist now - if not, I need to see the error
397        Wave/T/Z tw = $(ksBaseDFPath+folderStr+":"+path)
398       
399        if(WaveExists(tw))
400       
401        //      if(strlen(tw[0]) != num)
402        //              Print "string is not the specified length"
403        //      endif
404               
405                return(tw[0])
406        else
407                return("The specified wave does not exist: " + path)
408        endif
409End
410
411
412
413///////////////////////////////
414
415//
416//Write Wave 'wav' to hdf5 file 'fname'
417//Based on code from ANSTO (N. Hauser. nha 8/1/09)
418//
419// TODO:
420// -- figure out if this will write in the native format of the
421//     wave as passed in, or if it will only write as DP.
422// -- do I need to write separate functions for real, integer, etc.?
423//     
424// -- change the /P=home to the user-defined data path (which may be home)             
425//
426Function V_WriteWaveToHDF(fname, groupName, varName, wav)
427        String fname, groupName, varName
428        Wave wav
429       
430        variable err=0, fileID,groupID
431        String cDF = getDataFolder(1), temp
432        String NXentry_name
433       
434        try     
435                HDF5OpenFile/P=home /Z fileID  as fname  //open file read-write
436                if(!fileID)
437                        err = 1
438                        abort "HDF5 file does not exist"
439                endif
440               
441                //get the NXentry node name
442                HDF5ListGroup /TYPE=1 fileID, "/"
443                //remove trailing ; from S_HDF5ListGroup
444               
445                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
446               
447                NXentry_name = S_HDF5ListGroup
448                NXentry_name = ReplaceString(";",NXentry_name,"")
449                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
450                        err = 1
451                        abort "More than one entry under the root node. Ambiguous"
452                endif
453                //concatenate NXentry node name and groupName   
454                // SRK - NOV2015 - dropped this and require the full group name passed in
455//              groupName = "/" + NXentry_name + groupName
456                Print "groupName = ",groupName
457                HDF5OpenGroup /Z fileID , groupName, groupID
458
459                if(!groupID)
460                // don't create the group it the name isn't right -- throw up an error
461                        //HDF5CreateGroup /Z fileID, groupName, groupID
462                        err = 1
463                        HDF5CloseFile /Z fileID
464                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
465                        return(err)
466                else
467                        // get attributes and save them
468                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
469                        //Wave attributes = S_HDF5ListAttributes
470                endif
471       
472                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
473                if (V_flag != 0)
474                        err = 1
475                        abort "Cannot save wave to HDF5 dataset " + varName
476                endif   
477               
478               
479                //attributes - something could be added here as optional parameters and flagged
480//              String attributes = "units"
481//              Make/O/T/N=1 tmp
482//              tmp[0] = "dimensionless"
483//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
484//              if (V_flag != 0)
485//                      err = 1
486//                      abort "Cannot save attributes to HDF5 dataset"
487//              endif   
488        catch
489
490        endtry
491
492// it is not necessary to close the group here. HDF5CloseFile will close the group as well     
493        if(groupID)
494                HDF5CloseGroup /Z groupID
495        endif
496       
497        if(fileID)
498                HDF5CloseFile /Z fileID
499        endif
500
501        setDataFolder $cDF
502        return err
503end
504
505//Write Wave 'wav' to hdf5 file 'fname'
506//Based on code from ANSTO (N. Hauser. nha 8/1/09)
507//
508// TODO
509//
510// -- change the /P=home to the user-defined data path (which may be home)             
511//
512Function V_WriteTextWaveToHDF(fname, groupName, varName, wav)
513        String fname, groupName, varName
514        Wave/T wav
515       
516        variable err=0, fileID,groupID
517        String cDF = getDataFolder(1), temp
518        String NXentry_name
519       
520        try     
521                HDF5OpenFile/P=home /Z fileID  as fname  //open file read-write
522                if(!fileID)
523                        err = 1
524                        abort "HDF5 file does not exist"
525                endif
526               
527                //get the NXentry node name
528                HDF5ListGroup /TYPE=1 fileID, "/"
529                //remove trailing ; from S_HDF5ListGroup
530               
531                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
532               
533                NXentry_name = S_HDF5ListGroup
534                NXentry_name = ReplaceString(";",NXentry_name,"")
535                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
536                        err = 1
537                        abort "More than one entry under the root node. Ambiguous"
538                endif
539
540                //concatenate NXentry node name and groupName
541                // SRK - NOV2015 - dropped this and require the full group name passed in
542//              groupName = "/" + NXentry_name + groupName
543                Print "groupName = ",groupName
544
545                HDF5OpenGroup /Z fileID , groupName, groupID
546
547                if(!groupID)
548                // don't create the group it the name isn't right -- throw up an error
549                        //HDF5CreateGroup /Z fileID, groupName, groupID
550                        err = 1
551                        HDF5CloseFile /Z fileID
552                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
553                        return(err)
554                else
555                        // get attributes and save them
556                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
557                        //Wave attributes = S_HDF5ListAttributes
558                endif
559       
560                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
561                if (V_flag != 0)
562                        err = 1
563                        abort "Cannot save wave to HDF5 dataset " + varName
564                endif   
565               
566               
567                //attributes - something could be added here as optional parameters and flagged
568//              String attributes = "units"
569//              Make/O/T/N=1 tmp
570//              tmp[0] = "dimensionless"
571//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
572//              if (V_flag != 0)
573//                      err = 1
574//                      abort "Cannot save attributes to HDF5 dataset"
575//              endif   
576        catch
577
578        endtry
579       
580        if(groupID)
581                HDF5CloseGroup /Z groupID
582        endif
583       
584        if(fileID)
585                HDF5CloseFile /Z fileID
586        endif
587
588        setDataFolder $cDF
589        return err
590end
591
592//////////////////////////////
593//////////////////////////////
594//////////////////////////////
595
596Function V_KillNamedDataFolder(fname)
597        String fname
598       
599        Variable err=0
600       
601        String folderStr = V_GetFileNameFromPathNoSemi(fname)
602        folderStr = V_RemoveDotExtension(folderStr)
603       
604        KillDataFolder/Z $(ksBaseDFPath+folderStr)
605        err = V_flag
606       
607        return(err)
608end
609
610//given a filename of a SANS data filename of the form
611// name.anything
612//returns the name as a string without the ".fbdfasga" extension
613//
614// returns the input string if a"." can't be found (maybe it wasn't there"
615Function/S V_RemoveDotExtension(item)
616        String item
617        String invalid = item   //
618        Variable num=-1
619       
620        //find the "dot"
621        String runStr=""
622        Variable pos = strsearch(item,".",0)
623        if(pos == -1)
624                //"dot" not found
625                return (invalid)
626        else
627                //found, get all of the characters preceeding it
628                runStr = item[0,pos-1]
629                return (runStr)
630        Endif
631End
632
633//returns a string containing filename (WITHOUT the ;vers)
634//the input string is a full path to the file (Mac-style, still works on Win in IGOR)
635//with the folders separated by colons
636//
637// called by MaskUtils.ipf, ProtocolAsPanel.ipf, WriteQIS.ipf
638//
639Function/S V_GetFileNameFromPathNoSemi(fullPath)
640        String fullPath
641       
642        Variable offset1,offset2
643        String filename=""
644        //String PartialPath
645        offset1 = 0
646        do
647                offset2 = StrSearch(fullPath, ":", offset1)
648                if (offset2 == -1)                              // no more colons ?
649                        fileName = FullPath[offset1,strlen(FullPath) ]
650                        //PartialPath = FullPath[0, offset1-1]
651                        break
652                endif
653                offset1 = offset2+1
654        while (1)
655       
656        //remove version number from name, if it's there - format should be: filename;N
657        filename =  StringFromList(0,filename,";")              //returns null if error
658       
659        Return filename
660End
Note: See TracBrowser for help on using the repository browser.