source: sans/Dev/trunk/NCNR_User_Procedures/Reduction/VSANS/V_HDF5_RW_Utils.ipf @ 976

Last change on this file since 976 was 976, checked in by srkline, 6 years ago

Folder utilities to be able to properly move data between work folders (Duplicate w/ recursion)

Utilities to load in raw data from the main panel

Crude display of the raw data (fake, wrong dimensions) in a tabbed data panel

File size: 16.6 KB
Line 
1#pragma rtGlobals=3             // Use modern global access method and strict wave access.
2
3//
4// The base functions for R/W from HDF5 files.
5// All of the specific "get" and "write" functions call these base functions which are responsible
6// for all of the open/close mechanics.
7//
8// These VSANS file-specific functions are in:
9// V_HDF5_Read.ipf
10//               and
11// V_HDF5_Write.ipf
12//
13
14// data is read into:
15//      NewDataFolder/O/S root:Packages:NIST:VSANS:RawVSANS
16// so that the data folders of the raw data won't be lying around on the top level, looking like
17// 1D data sets and interfering.
18
19
20// the base data folder path where the raw data is loaded
21Strconstant ksBaseDFPath = "root:Packages:NIST:VSANS:RawVSANS:"
22
23
24
25// passing null file string presents a dialog
26Proc Read_HDF5_Raw_No_Attributes()
27        V_LoadHDF5Data("","RAW")
28End
29
30// TODO:
31//  x- move the initializtion of the raw data folder to be in the as-yet unwritten initialization routine for
32// reduction. be sure that it's duplicated in the VCALC initialization too.
33// -- as needed, get rid of the FAKE redimension of the data from 3D->2D and from 128x128 to something else for VSANS
34//    This is a fake since I don't have anything close to correct fake data yet. (1/29/16)
35//
36Function V_LoadHDF5Data(file,folder)
37        String file,folder
38
39        String base_name
40        SetDataFolder $("root:Packages:NIST:VSANS:"+folder)
41//      SetDataFolder root:
42        if(cmpstr(folder,"RAW")==0)
43                base_name="entry"
44        else
45                base_name=""
46        endif
47       
48        Variable err= V_LoadHDF5_NoAtt(file,base_name)  // reads into current folder
49       
50        // if RAW data, then generate the errors and linear data copy
51        // do this 9x
52        string tmpStr = "root:Packages:NIST:VSANS:RAW:entry:entry:instrument:"
53        if(cmpstr(folder,"RAW")==0)
54                V_MakeDataError(tmpStr+"detector_B")
55                V_MakeDataError(tmpStr+"detector_MB")
56                V_MakeDataError(tmpStr+"detector_MT")
57                V_MakeDataError(tmpStr+"detector_ML")
58                V_MakeDataError(tmpStr+"detector_MR")
59                V_MakeDataError(tmpStr+"detector_FB")
60                V_MakeDataError(tmpStr+"detector_FT")
61                V_MakeDataError(tmpStr+"detector_FL")
62                V_MakeDataError(tmpStr+"detector_FR")
63
64// TODO -- once I get "real" data, get rid of this call to force the data to be proper dimensions.
65                V_RedimFakeData()
66               
67/// END FAKE DATA CORRECTIONS           
68               
69        endif
70       
71        SetDataFolder root:
72        return(err)
73End
74
75//
76// TODO -- this is all FAKED since all the data arrays are (1,128,128)
77// I'm intentionally using the wrong number of pixels so I'm more likely to go back and get rid of this later.
78//
79// the SetScale parts may be useful later.
80//
81Function V_RedimFakeData()
82       
83                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_B
84                Wave det_B=data
85                Redimension/N=(300,300)/E=1 det_B
86               
87                Variable ctr=20,npix=100
88                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MT
89                Wave det_MT=data
90                Redimension/N=(npix,48)/E=1 det_MT             
91                SetScale/I x -npix/2,npix/2,"",det_MT
92                SetScale/I y ctr,ctr+48,"",det_MT
93
94                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MB
95                Wave det_MB=data
96                Redimension/N=(npix,48)/E=1 det_MB             
97                SetScale/I x -npix/2,npix/2,"",det_MB
98                SetScale/I y -ctr,-ctr-48,"",det_MB
99               
100                ctr=30
101                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_ML
102                Wave det_ML=data
103                Redimension/N=(48,npix)/E=1 det_ML             
104                SetScale/I x -ctr-48,-ctr,"",det_ML
105                SetScale/I y -npix/2,npix/2,"",det_ML
106               
107                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_MR
108                Wave det_MR=data
109                Redimension/N=(48,npix)/E=1 det_MR             
110                SetScale/I x ctr,ctr+48,"",det_MR
111                SetScale/I y -npix/2,npix/2,"",det_MR
112               
113                ctr=30
114                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FT
115                Wave det_FT=data
116                Redimension/N=(npix,48)/E=1 det_FT             
117                SetScale/I x -npix/2,npix/2,"",det_FT
118                SetScale/I y ctr,ctr+48,"",det_FT
119
120                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FB
121                Wave det_FB=data
122                Redimension/N=(npix,48)/E=1 det_FB             
123                SetScale/I x -npix/2,npix/2,"",det_FB
124                SetScale/I y -ctr,-ctr-48,"",det_FB
125               
126                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FL
127                Wave det_FL=data
128                Redimension/N=(48,npix)/E=1 det_FL             
129                SetScale/I x -ctr-48,-ctr,"",det_FL
130                SetScale/I y -npix/2,npix/2,"",det_FL
131               
132                SetDataFolder root:Packages:NIST:VSANS:RAW:entry:entry:instrument:detector_FR
133                Wave det_FR=data
134                Redimension/N=(48,npix)/E=1 det_FR             
135                SetScale/I x ctr,ctr+48,"",det_FR
136                SetScale/I y -npix/2,npix/2,"",det_FR
137
138
139        return(0)
140End
141
142
143// This loads for speed, since loading the attributes takes a LOT of time.
144//
145// this will load in the whole HDF file all at once.
146// Attributes are NOT loaded at all.
147//
148// TODO: remove the P=home restriction top make this more generic
149// -- get rid of bits leftover here that I don't need
150// -- be sure I'm using all of the correct flags in the HDF5LoadGroup operation
151// -- settle on how the base_name is to be used. "entry" for the RAW, fileName for the "rawVSANS"?
152//
153// passing in "" for base_name will take the name from the file name as selected
154//
155Function V_LoadHDF5_NoAtt(fileName,base_name)
156        String fileName, base_name
157       
158//      if ( ParamIsDefault(hdf5Path) )
159//              hdf5Path = "/"
160//      endif
161
162        String hdf5path = "/"           //always read from the top
163        String status = ""
164
165        Variable fileID = 0
166        HDF5OpenFile/R/P=home/Z fileID as fileName              //read file from home directory?
167//      HDF5OpenFile/R/P=catPathName/Z fileID as fileName
168        if (V_Flag != 0)
169                return 0
170        endif
171
172        String/G root:file_path = S_path
173        String/G root:file_name = S_FileName
174       
175        if ( fileID == 0 )
176                Print fileName + ": could not open as HDF5 file"
177                return (0)
178        endif
179       
180//s_tic()               //fast
181       
182        SVAR tmpStr=root:file_name
183        fileName=tmpStr         //SRK - in case the file was chosen from a dialog, I'll need access to the name later
184       
185        //   read the data (too bad that HDF5LoadGroup does not read the attributes)
186        if(cmpstr(base_name,"") == 0)
187                base_name = StringFromList(0,FileName,".")
188        endif
189        //base_name = "entry"
190       
191        HDF5LoadGroup/Z/L=7/O/R/T=$base_name  :, fileID, hdf5Path               //      recursive
192        if ( V_Flag != 0 )
193                Print fileName + ": could not open as HDF5 file"
194                setdatafolder root:
195                return (0)
196        endif
197
198        HDF5CloseFile fileID
199       
200//s_toc()
201        return(0)
202end     
203
204
205// read a single real value
206// - fname passed in is the full path to the file on disk
207// - path is the path to the value in the HDF tree
208//
209// check to see if the value exists (It will be a wave)
210// -- if it does, return the value from the local folder
211// -- if not, read the file in, then return the value
212//
213Function V_getRealValueFromHDF5(fname,path)
214        String fname,path
215
216        String folderStr=""
217        Variable valExists=0
218       
219        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
220       
221        if(Exists(ksBaseDFPath+folderStr+":"+path))
222                valExists=1
223        endif
224       
225        if(!valExists)
226                //then read in the file
227                V_LoadHDF5_NoAtt(fname,"")
228        endif
229
230// this should exist now - if not, I need to see the error
231        Wave/Z w = $(ksBaseDFPath+folderStr+":"+path)
232       
233        if(WaveExists(w))
234                return(w[0])
235        else
236                return(-999999)
237        endif   
238End
239
240// Returns a wave reference, not just a single value
241// ---then you pick what you need from the wave
242//
243// - fname passed in is the full path to the file on disk
244// - path is the path to the value in the HDF tree
245//
246// check to see if the value exists (It will be a wave)
247// -- if it does, return the value from the local folder
248// -- if not, read the file in, then return the value
249//
250Function/WAVE V_getRealWaveFromHDF5(fname,path)
251        String fname,path
252
253        String folderStr=""
254        Variable valExists=0
255       
256        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
257       
258        if(Exists(ksBaseDFPath+folderStr+":"+path))
259                valExists=1
260        endif
261       
262        if(!valExists)
263                //then read in the file
264                V_LoadHDF5_NoAtt(fname,"")
265        endif
266
267// this should exist now - if not, I need to see the error
268        Wave wOut = $(ksBaseDFPath+folderStr+":"+path)
269       
270        return wOut
271       
272End
273
274// Returns a wave reference, not just a single value
275// ---then you pick what you need from the wave
276//
277// - fname passed in is the full path to the file on disk
278// - path is the path to the value in the HDF tree
279//
280// check to see if the value exists (It will be a wave)
281// -- if it does, return the value from the local folder
282// -- if not, read the file in, then return the value
283//
284Function/WAVE V_getTextWaveFromHDF5(fname,path)
285        String fname,path
286
287        String folderStr=""
288        Variable valExists=0
289       
290        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
291       
292        if(Exists(ksBaseDFPath+folderStr+":"+path))
293                valExists=1
294        endif
295       
296        if(!valExists)
297                //then read in the file
298                V_LoadHDF5_NoAtt(fname,"")
299        endif
300
301// this should exist now - if not, I need to see the error
302        Wave/T wOut = $(ksBaseDFPath+folderStr+":"+path)
303       
304        return wOut
305       
306End
307
308
309//
310//   TODO
311// depricated? in HDF5 - store all of the values as real?
312// Igor sees no difference in real and integer variables (waves are different)
313// BUT-- Igor 7 will have integer variables
314//
315// truncate to integer before returning??
316//
317//  TODO
318// write a "getIntegerWave" function??
319//
320//////  integer values
321// reads 32 bit integer
322Function V_getIntegerFromHDF5(fname,path)
323        String fname                            //full path+name
324        String path                             //path to the hdf5 location
325       
326        Variable val = V_getRealValueFromHDF5(fname,path)
327       
328        val = round(val)
329        return(val)
330End
331
332
333// read a single string
334// - fname passed in is the full path to the file on disk
335// - path is the path to the value in the HDF tree
336// - num is the number of characters in the VAX string
337// check to see if the value exists (It will be a wave)
338// -- if it does, return the value from the local folder
339// -- if not, read the file in, then return the value
340//
341// TODO -- string could be checked for length, but returned right or wrong
342//
343Function/S V_getStringFromHDF5(fname,path,num)
344        String fname,path
345        Variable num
346
347        String folderStr=""
348        Variable valExists=0
349       
350        folderStr = V_RemoveDotExtension(V_GetFileNameFromPathNoSemi(fname))
351       
352        if(Exists(ksBaseDFPath+folderStr+":"+path))
353                valExists=1
354        endif
355       
356        if(!valExists)
357                //then read in the file
358                V_LoadHDF5_NoAtt(fname,"")
359        endif
360
361// this should exist now - if not, I need to see the error
362        Wave/T/Z tw = $(ksBaseDFPath+folderStr+":"+path)
363       
364        if(WaveExists(tw))
365       
366        //      if(strlen(tw[0]) != num)
367        //              Print "string is not the specified length"
368        //      endif
369               
370                return(tw[0])
371        else
372                return("The specified wave does not exist: " + path)
373        endif
374End
375
376
377
378///////////////////////////////
379
380//
381//Write Wave 'wav' to hdf5 file 'fname'
382//Based on code from ANSTO (N. Hauser. nha 8/1/09)
383//
384// TODO:
385// -- figure out if this will write in the native format of the
386//     wave as passed in, or if it will only write as DP.
387// -- do I need to write separate functions for real, integer, etc.?
388//     
389// -- change the /P=home to the user-defined data path (which may be home)             
390//
391Function V_WriteWaveToHDF(fname, groupName, varName, wav)
392        String fname, groupName, varName
393        Wave wav
394       
395        variable err=0, fileID,groupID
396        String cDF = getDataFolder(1), temp
397        String NXentry_name
398       
399        try     
400                HDF5OpenFile/P=home /Z fileID  as fname  //open file read-write
401                if(!fileID)
402                        err = 1
403                        abort "HDF5 file does not exist"
404                endif
405               
406                //get the NXentry node name
407                HDF5ListGroup /TYPE=1 fileID, "/"
408                //remove trailing ; from S_HDF5ListGroup
409               
410                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
411               
412                NXentry_name = S_HDF5ListGroup
413                NXentry_name = ReplaceString(";",NXentry_name,"")
414                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
415                        err = 1
416                        abort "More than one entry under the root node. Ambiguous"
417                endif
418                //concatenate NXentry node name and groupName   
419                // SRK - NOV2015 - dropped this and require the full group name passed in
420//              groupName = "/" + NXentry_name + groupName
421                Print "groupName = ",groupName
422                HDF5OpenGroup /Z fileID , groupName, groupID
423
424                if(!groupID)
425                // don't create the group it the name isn't right -- throw up an error
426                        //HDF5CreateGroup /Z fileID, groupName, groupID
427                        err = 1
428                        HDF5CloseFile /Z fileID
429                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
430                        return(err)
431                else
432                        // get attributes and save them
433                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
434                        //Wave attributes = S_HDF5ListAttributes
435                endif
436       
437                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
438                if (V_flag != 0)
439                        err = 1
440                        abort "Cannot save wave to HDF5 dataset " + varName
441                endif   
442               
443               
444                //attributes - something could be added here as optional parameters and flagged
445//              String attributes = "units"
446//              Make/O/T/N=1 tmp
447//              tmp[0] = "dimensionless"
448//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
449//              if (V_flag != 0)
450//                      err = 1
451//                      abort "Cannot save attributes to HDF5 dataset"
452//              endif   
453        catch
454
455        endtry
456
457// it is not necessary to close the group here. HDF5CloseFile will close the group as well     
458        if(groupID)
459                HDF5CloseGroup /Z groupID
460        endif
461       
462        if(fileID)
463                HDF5CloseFile /Z fileID
464        endif
465
466        setDataFolder $cDF
467        return err
468end
469
470//Write Wave 'wav' to hdf5 file 'fname'
471//Based on code from ANSTO (N. Hauser. nha 8/1/09)
472//
473// TODO
474//
475// -- change the /P=home to the user-defined data path (which may be home)             
476//
477Function V_WriteTextWaveToHDF(fname, groupName, varName, wav)
478        String fname, groupName, varName
479        Wave/T wav
480       
481        variable err=0, fileID,groupID
482        String cDF = getDataFolder(1), temp
483        String NXentry_name
484       
485        try     
486                HDF5OpenFile/P=home /Z fileID  as fname  //open file read-write
487                if(!fileID)
488                        err = 1
489                        abort "HDF5 file does not exist"
490                endif
491               
492                //get the NXentry node name
493                HDF5ListGroup /TYPE=1 fileID, "/"
494                //remove trailing ; from S_HDF5ListGroup
495               
496                Print "S_HDF5ListGroup = ",S_HDF5ListGroup
497               
498                NXentry_name = S_HDF5ListGroup
499                NXentry_name = ReplaceString(";",NXentry_name,"")
500                if(strsearch(NXentry_name,":",0)!=-1) //more than one entry under the root node
501                        err = 1
502                        abort "More than one entry under the root node. Ambiguous"
503                endif
504
505                //concatenate NXentry node name and groupName
506                // SRK - NOV2015 - dropped this and require the full group name passed in
507//              groupName = "/" + NXentry_name + groupName
508                Print "groupName = ",groupName
509
510                HDF5OpenGroup /Z fileID , groupName, groupID
511
512                if(!groupID)
513                // don't create the group it the name isn't right -- throw up an error
514                        //HDF5CreateGroup /Z fileID, groupName, groupID
515                        err = 1
516                        HDF5CloseFile /Z fileID
517                        DoAlert 0, "HDF5 group does not exist "+groupName+varname
518                        return(err)
519                else
520                        // get attributes and save them
521                        //HDF5ListAttributes /Z fileID, groupName    this is returning null. expect it to return semicolon delimited list of attributes
522                        //Wave attributes = S_HDF5ListAttributes
523                endif
524       
525                HDF5SaveData /O /Z /IGOR=0 wav, groupID, varName
526                if (V_flag != 0)
527                        err = 1
528                        abort "Cannot save wave to HDF5 dataset " + varName
529                endif   
530               
531               
532                //attributes - something could be added here as optional parameters and flagged
533//              String attributes = "units"
534//              Make/O/T/N=1 tmp
535//              tmp[0] = "dimensionless"
536//              HDF5SaveData /O /Z /IGOR=0 /A=attributes tmp, groupID, varName
537//              if (V_flag != 0)
538//                      err = 1
539//                      abort "Cannot save attributes to HDF5 dataset"
540//              endif   
541        catch
542
543        endtry
544       
545        if(groupID)
546                HDF5CloseGroup /Z groupID
547        endif
548       
549        if(fileID)
550                HDF5CloseFile /Z fileID
551        endif
552
553        setDataFolder $cDF
554        return err
555end
556
557//////////////////////////////
558//////////////////////////////
559//////////////////////////////
560
561Function V_KillNamedDataFolder(fname)
562        String fname
563       
564        Variable err=0
565       
566        String folderStr = V_GetFileNameFromPathNoSemi(fname)
567        folderStr = V_RemoveDotExtension(folderStr)
568       
569        KillDataFolder/Z $(ksBaseDFPath+folderStr)
570        err = V_flag
571       
572        return(err)
573end
574
575//given a filename of a SANS data filename of the form
576// name.anything
577//returns the name as a string without the ".fbdfasga" extension
578//
579// returns the input string if a"." can't be found (maybe it wasn't there"
580Function/S V_RemoveDotExtension(item)
581        String item
582        String invalid = item   //
583        Variable num=-1
584       
585        //find the "dot"
586        String runStr=""
587        Variable pos = strsearch(item,".",0)
588        if(pos == -1)
589                //"dot" not found
590                return (invalid)
591        else
592                //found, get all of the characters preceeding it
593                runStr = item[0,pos-1]
594                return (runStr)
595        Endif
596End
597
598//returns a string containing filename (WITHOUT the ;vers)
599//the input string is a full path to the file (Mac-style, still works on Win in IGOR)
600//with the folders separated by colons
601//
602// called by MaskUtils.ipf, ProtocolAsPanel.ipf, WriteQIS.ipf
603//
604Function/S V_GetFileNameFromPathNoSemi(fullPath)
605        String fullPath
606       
607        Variable offset1,offset2
608        String filename=""
609        //String PartialPath
610        offset1 = 0
611        do
612                offset2 = StrSearch(fullPath, ":", offset1)
613                if (offset2 == -1)                              // no more colons ?
614                        fileName = FullPath[offset1,strlen(FullPath) ]
615                        //PartialPath = FullPath[0, offset1-1]
616                        break
617                endif
618                offset1 = offset2+1
619        while (1)
620       
621        //remove version number from name, if it's there - format should be: filename;N
622        filename =  StringFromList(0,filename,";")              //returns null if error
623       
624        Return filename
625End
Note: See TracBrowser for help on using the repository browser.