pro drops2_to_dprain,network,year,month,day,DO_GRAPHICS=do_graphics, $
                         APPLY_BLOCKAGE=apply_blockage,DPQC=dpqc,     $
                         POSTSCRIPT=postscript,field_campaign=field_campaign, $
                         scantype=scantype
;
; *** Program to run mainline code processing radar data
; *** to generate DROPS2, Rain Rates, HID, DSD, and associated imagery.
; *** Modified by Jason Pippitt 04/2015 to run DROPS2 
; *** 05/2018 update:  output to cfradial,UF could not handle the amount of output fields
;
    COMMON misc,TRUE,FALSE
    COMMON dirs,img_dir,uf_dir,hid_img_dir,hid_dir,dsd_dir
    COMMON graphics,USE_Z_BUFFER,TC,XS,YS,cols,rows
    COMMON graphic_parms,scale,bottom,ncolors
    COMMON my_colors,black,purple,blue,cyan,green,yellow,orange,red,white
    COMMON symbols,plot,asterisk,dot,diamond,triangle,square,exx
    COMMON linestyles,solid,dotted,dashed,dashdot,dashdotdot,longdash

    spawn,'date',beg_proc_time
    TRUE=1 & FALSE=0
    !QUIET=TRUE
    USE_Z_BUFFER = FALSE
    xs=499 & ys=499
;
; *** Deal with keywords
;
    if(KEYWORD_SET(field_campaign)) then field_campaign=strupcase(field_campaign)
    if(NOT KEYWORD_SET(APPLY_BLOCKAGE)) then apply_blockage=FALSE
    if(NOT KEYWORD_SET(DPQC))           then dpqc=FALSE
    if(NOT KEYWORD_SET(DO_GRAPHICS))    then do_graphics=FALSE    
    if(NOT KEYWORD_SET(POSTSCRIPT))     then begin
        postscript=FALSE
    endif else begin
        do_Graphics=TRUE
    endelse

    CASE strupcase(network) of
               'GPMVN': BEGIN
	                 sites = ['KABR','KAKQ','KAMX','KAPX','KARX','KBMX','KBOX',$
                         'KBRO','KBUF','KBYX','KCCX','KCLX','KCRP','KDDC',$
                         'KDGX','KDLH','KDMX','KDOX','KDVN','KEAX','KEVX',$
                         'KFSD','KFTG','KFWS','KGRK','KGRR','KHGX','KHTX',$
                         'KICT','KILN','KILX','KINX','KIWX','KJAX','KJGX',$
                         'KJKL','KLCH','KLGX','KLIX','KLOT','KLSX','KLZK',$
                         'KMHX','KMKX','KMLB','KMOB','KMQT','KMVX','KNQA',$
                         'KOKX','KPAH','KRAX','KSGF','KSHV','KSRX','KTBW',$
                         'KTLH','KTLX','KTWX','KTYX','KWAJ','PAEC','KMXX',$
                         'PAIH','PGUA','PHKI','PHMO','TJUA','CHILL','KCAE', $
                         'KGSP','KING','KMRX','AL1','JG1','MC1','NT1','PE1', $
                         'SF1','ST1','SV1','TM1','KHPX','KEOX','KVNX','KPOE', $
                         'KVAX','KOHX','KFFC','KMUX','KMAX','KBBX','RODN','KGWX']
;                sites = ['KPOE']
                END

	       'GPMVNNPOL': BEGIN
                  sites = ['NPOL']
                END

	       'GPMVNPAST': BEGIN
	         ;sites = ['KEOX','KFFC','KHPX','KOHX','KPOE','KVAX','KVNX']
		 sites = ['KDDC','KRAX','KVNX']
	       END

                'REUNION': BEGIN
                  sites = ['REUNION']
                END

               'TRMMVN': BEGIN
                  sites = ['KAMX','KBMX','KBRO','KBYX','KCLX','KCRP','KDGX','KEVX', $
	                   'KFWS','KGRK','KHGX','KHTX','KJAX','KJGX','KLCH','KLIX', $
             	           'KMLB','KMOB','KSHV','KTBW','KTLH','KWAJ']
                END

                'GCOMW1': BEGIN
;                  sites = ['KLGX','KWAJ','PAIH']
;                  sites = ['KWAJ','PAIH']
                   sites = ['KHGX']
                END

                'F16': BEGIN
;                   sites = ['KWAJ','PAIH']
                    sites = ['KHGX']
                END
 
                'F17': BEGIN
;                   sites = ['KWAJ','PAIH']
                    sites = ['KHGX']
                END

	        'F18': BEGIN
;                   sites = ['KWAJ','PAIH']
                    sites = ['KHGX']
                END

                'METOPA': BEGIN
;                   sites = ['KWAJ','PAIH']
                    sites = ['KHGX']
                END

                'METOPB': BEGIN
;                   sites = ['KWAJ','PAIH']
                    sites = ['KHGX']
                END

                'NOAA18': BEGIN
;                   sites = ['KWAJ','PAIH']
                    sites = ['KHGX']
                END

                'NOAA19': BEGIN
;                   sites = ['KWAJ','PAIH']
                    sites = ['KHGX']
                END

                'NPP': BEGIN
;                   sites = ['KWAJ','PAIH']
                    sites = ['KHGX']
                END

                'KMLB': BEGIN
                  sites = ['KMLB']
                END

		'KHGX': BEGIN
		  sites = ['KHGX']
		END
                
                'KWAJ': BEGIN
                  sites = ['KWAJ']
                END

                'KDOX': BEGIN
                  sites = ['KDOX']
                END
                
                'KAKQ': BEGIN
                  sites = ['KAKQ']
                END
                
                'NPOL': BEGIN
                  sites = ['NPOL']
                END
                
                'KLGX': BEGIN
                  sites = ['KLGX']
                END
 
                'KARX': BEGIN
                  sites = ['KARX']
                END
 
                'KLWX': BEGIN
                  sites = ['KLWX']
                END

		'PHMO': BEGIN
                  sites = ['PHMO']
                END

		'PHKI': BEGIN
                  sites = ['PHKI']
                END

		'PHKM': BEGIN
                   sites = ['PHKM']
                END

		'PHWA': BEGIN
                  sites = ['PHWA']
                END

                'KLTX': BEGIN
                  sites = ['KLTX']
                END

                'KRAX': BEGIN
                  sites = ['KRAX']
                END

	        'KMHX': BEGIN
                  sites = ['KMHX']
                END

	        'KEVX': BEGIN
                  sites = ['KEVX']
                END

		'KMOB': BEGIN
                   sites = ['KMOB']
                END

	        'KMXX': BEGIN
                  sites = ['KMXX']
                END

	        'KJGX': BEGIN
                  sites = ['KJGX']
                END
  
                'KEWX': BEGIN
                  sites = ['KEWX']
                END

                'KGRK': BEGIN
                  sites = ['KGRK']
                END

	        'KLCH': BEGIN
                  sites = ['KLCH']
                END

	        'KPOE': BEGIN
                  sites = ['KPOE']
                END
                
                ELSE: BEGIN
                 print,'Site not supported! ' + network
                END
                
            ENDCASE
    
    ns = n_elements(sites)
    nt = 0
    na = 0
    nb = 0

;
; *** Load the blockage map that has already been saved in an
; *** IDL save file
;
    if(APPLY_BLOCKAGE) then begin
       mask_file = 'Masks/' + field_campaign + '_Blockage_mask.sav'
       restore,mask_file
    endif
;
; *** Set date to starings
;
    year  = string(year,format='(I4.4)')
    syear = strmid(year,2,2)
    month = string(month,format='(I2.2)')
    day   = string(day,format='(I2.2)')
    the_date = month + day
;
; ** Setup output file
;
   output_file_dir = 'output/' + year + '/'
   spawn, 'mkdir -p ' + output_file_dir
   if(network eq 'NPOL') then begin
      output_file = output_file_dir + network + '_' + scantype + '_' + the_date + '.txt
   endif else begin
      output_file = output_file_dir + network + '_' + the_date + '.txt'
   endelse
   openw,unit,output_file,/get_lun

   for s=0L,ns-1 do begin
        site = sites(s)
        radar_name = sites(s)
;
; *** Data directories
;
    files = get_filelist2(network,site,year,month,day,in_dir,out_dir,multi_out,DPQC=dpqc,field_campaign=field_campaign,scantype=scantype)
    if(files[0] eq '') then begin
       print
       printf,unit
       print,'****** No data for:  ' + site + ' on ' + month + '/' + day + '/' + year + ' ******'
       print
       printf,unit,'****** No data for:  ' + site + ' on ' + month + '/' + day + '/' + year + ' ******'
       printf,unit
       goto,next_site
    endif

    print,'In dir:  ' + in_dir
    print,'Out dir: ' + out_dir
    print,'Multi dir: ' + multi_out
    printf,unit,'In dir:  ' + in_dir
    printf,unit,'Out dir: ' + out_dir
    printf,unit,'Multi dir: ' + multi_out
;
; *** Get a list of input file
;
    wc = in_dir + '*_KDP.cf.gz'
    files = file_search(wc,COUNT=nf)
    if(nf eq 0) then begin
        print,'No files found in ' + wc
        printf,unit,'No files found in ' + wc
        stop
    endif

    nt = nt + nf
;
; *** Temporary dir for storing files
;
    tmp_dir= 'tmp_' + network + '_' + year + '_' + month + day + '/'
    spawn,'mkdir -p ' + tmp_dir
;
; *** Process files for the day
;
    for i=0,nf-1 do begin
        file = files[i]

        fileb = file_basename(file,'.cf.gz')
        filebcf = file_basename(file,'_KDP.cf.gz')
        print
        print,'Input File:'
        print,'<-- ' + file
        printf,unit
        printf,unit,'Input File:'
        printf,unit,'<-- ' + file

        a = strsplit(fileb,'.',/extract)
        org_file_base = a[0]

        radar = rsl_anyformat_to_radar(file,/quiet,ERROR=error)
        if(ERROR) then begin
            print,'Error loading radar: ' + file
            printf,unit,'Error loading radar: ' + file
            stop
            goto,next_file
        endif
;        print,'# of sweeps = ',radar.volume[0].h.nsweeps
        scan_type = strlowcase(radar.h.scan_mode)
        stype = radar.h.scan_mode
        if(strmid(scan_type,0,3) ne scan_type) then begin
            print,'Scan mode not supported! ' + scan_type
            printf,unit,'Scan mode not supported! ' + scan_type
            goto,next_file
        endif
;
; *** Apply blockage
;
        if(APPLY_BLOCKAGE) then begin
            print
            print,'Applying blockage correction to data...'
            printf,unit
            printf,unit,'Applying blockage correction to data...'

            CASE strupcase(field_campaign) of 
               'IFLOODS': BEGIN
                  print,'Calling adjust_ifloods_blockage_from_radar...'
                  printf,unit,'Calling adjust_ifloods_blockage_from_radar...'
                  flag = adjust_ifloods_blockage_from_radar(radar,m)
                END

               'IPHEX': BEGIN
                  print,'Calling adjust_iphex_blockage_from_radar...'
                  printf,unit,'Calling adjust_iphex_blockage_from_radar...'
                  flag = adjust_iphex_blockage_from_radar(radar,m)
                END

               'OLYMPEX': BEGIN
                  print,'Calling adjust_olympex_blockage_from_radar...'
                  printf,unit,'Calling adjust_olympex_blockage_from_radar...'
                  flag = adjust_olympex_blockage_from_radar(radar,m)
                END
            ENDCASE
            if(flag ne 'OK') then begin
                print,flag
                printf,unit,flag
;                goto,next_file
            endif
        endif
;
; *** Extract data and time from radar structure
;
;        help,/str,radar.h
        year   = string(radar.h.year,format='(I4.4)')
        month  = string(radar.h.month,format='(I2.2)')
        day    = string(radar.h.day,format='(I2.2)')
        hour   = string(radar.h.hour,format='(I2.2)')
        minute = string(radar.h.minute,format='(I2.2)')
        second = string(radar.h.sec,format='(I2.2)')
        the_date = month + day
;
; *** We will use archived RUC soundings
;
            mdays = [00,31,28,31,30,31,30,31,31,30,31,30,31]
;            sounding_dir = '/gvraid/trmmgv/Soundings/RUC_Soundings/' + year + '/' + the_date + '/' + site + '/'
            sounding_dir = '/gpmgv3/raw/Soundings/RUC_Soundings/' + year + '/' + the_date + '/' + site + '/'
            chour=hour
            
            if (site eq 'KWAJ') then begin
                raw_sounding_file = sounding_dir + site + '_' + year + '_' + month + day + '_00UTC.txt'
                goto,kwaj_snd
            endif
            if (site eq 'REUNION') then begin
                uwy_sounding_dir = '/gvraid/trmmgv/Soundings/UWY_Soundings/' + year + '/' + month + '/Reunion/'
                raw_sounding_file = uwy_sounding_dir + 'Reunion_FMEE_' + year + '_' + month + day + '_12UTC.txt'
                goto,kwaj_snd
            endif

            if(minute ge 30) then chour=hour+1
            if(chour eq 24) then begin
              chour = 00
              cday = day + 1
              if(cday gt mdays(month)) then begin
                 cmonth=month+1
                 cday = 01
                 if(cmonth gt 12) then begin
                    cmonth = 01
                    cday=01
                    cyear = year+1
                    year = string(cyear,format='(I4.4)')
                 endif
              month = string(cmonth,format='(I2.2)')
              day   = string(cday,format='(I2.2)')
              sounding_dir = '/gpmgv3/raw/Soundings/RUC_Soundings/' + year + $
                             '/' + month + day + '/' + site + '/'
;              sounding_dir = '/gvraid/trmmgv/Soundings/RUC_Soundings/' + year + $
;                             '/' + month + day + '/' + site + '/'
              endif
              day   = string(cday,format='(I2.2)')
;            sounding_dir = '/gvraid/trmmgv/Soundings/RUC_Soundings/' + year + '/' + month + day + '/' + site + '/'
            sounding_dir = '/gpmgv3/raw/Soundings/RUC_Soundings/' + year + '/' + month + day + '/' + site + '/'
            endif
            hour   = string(chour,format='(I2.2)')
            raw_sounding_file = sounding_dir + site + '_' + year + '_' + month + day + '_' + hour + 'UTC.txt'
kwaj_snd:
            sounding_file = tmp_dir + site + '_' + month + day + '_sounding.txt'
            c = "sed '1,2d' " + raw_sounding_file + ' > ' + sounding_file
            spawn,c
            print
            print,'Raw Sounding file --> ' + raw_sounding_file
            print,'Sounding --> ' + sounding_file
            print
            printf,unit
            printf,unit,'Raw Sounding file --> ' + raw_sounding_file
            printf,unit,'Sounding --> ' + sounding_file
            printf,unit
;
; *** UWY soundings
;
;        mdays = [00,31,28,31,30,31,30,31,31,30,31,30,31]
;        ZTIME='00'
;        if(hour*1 gt 6 and hour*1 le 18) then ZTIME='12'
;        if(hour*1 gt 18) then begin
;             ZTIME='00'
;             day = string(radar.h.day+1,format='(I2.2)')
;             if(radar.h.day+1 gt mdays(month)) then begin
;                 month = string(radar.h.month+1,format='(I2.2)')
;                 day = string(01,format='(I2.2)')
;             endif
;             if(radar.h.month+1 gt 12) then begin
;                 month = string(01,format='(I2.2)')
;                 year = string(radar.h.year+1,format='(I4.4)')
;                 day = string(01,format='(I2.2)')
;             endif
;        endif
;
; *** Name a new DROP UF file
; 
        print
        print,'Original Fields:  ',radar.volume.h.field_type
        print
        printf,unit
        printf,unit,'Original Fields:  ',radar.volume.h.field_type
        printf,unit
;
; *** Run HID program with radar
;
proc_hid:        
        print
        print,'Calculating HID...'
        printf,unit
        printf,unit,'Calculating HID...'
        flag = retrieve_hid(radar,SOUNDING_FILE=sounding_file,POSTSCRIPT=postscript)
        if(flag ne 'OK') then begin
            print,flag
            goto,next_file
        endif
        print
        print,'HID finished, current fields:  ', radar.volume.h.field_type
        printf,unit
        printf,unit,'HID finished, current fields:  ', radar.volume.h.field_type
;
; *** Now add the D0 and Nw fields to a new UF file.
;
    print
    print,'Adding DSD Fields...
    printf,unit
    printf,unit,'Adding DSD Fields...
    dsd_dir = out_dir

    flag = add_dsd_fields(radar,DPQC=dpqc,network=network,field_campaign=field_campaign,site=site)
    if(flag ne 'OK') then begin
        print,flag
        stop
        return
    endif
    print,'DSD finished, current fields:  ', radar.volume.h.field_type
    printf,unit,'DSD finished, current fields:  ', radar.volume.h.field_type

;
; *** Now use the returned radar structure to get the PolZR rain rates.
;
proc_polzr:
        print
        print,'Calculating DP Rain Estimates...'
        printf,unit
        printf,unit,'Calculating DP Rain Estimates...'
;
; *** Add's PolZR and Cifelli et al. 2002 rain volumes
;
        flag = retrieve_dp_rainrates(radar,radar_name,year,month,day)
        print,'DP Rain Estimates finished, current fields:  ', radar.volume.h.field_type
        print 
        printf,unit,'DP Rain Estimates finished, current fields:  ', radar.volume.h.field_type
        printf,unit
;        print,radar.volume.h.field_type
;
; *** Save the DROPS+HID+PolZR UF
;
        year   = string(radar.h.year,format='(I4.4)')
        month  = string(radar.h.month,format='(I2.2)')
        day    = string(radar.h.day,format='(I2.2)')

        pol_dir = out_dir
        spawn,'mkdir -p ' + pol_dir
;        pol_uf_file = pol_dir + fileb + '.gz'
        pol_cf_file = pol_dir + filebcf + '.cf'
       
        if (site eq 'KWAJ' or site eq 'NPOL') then begin
;	    my_fields = ['ZZ','CZ','DR','RH','PH','KD','SQ','SW','VR','SD', $
;                         'RR','RP','RC','D0','NW','FH','N2','DM','MC','MP','MR','HC']
            my_fields = ['ZZ','CZ','DR','RH','PH','KD','SW','VR', $
                         'RP','RC','NW','FH','DM']
            if (site eq 'KWAJ' and radar.h.scan_mode eq 'RHI') then $
	        my_fields = ['ZZ','CZ','DR','RH','PH','KD', $
                             'RP','RC','D0','NW','FH','N2','DM']
        endif else begin
           my_fields = ['ZZ','CZ','DR','RH','PH','KD','VR','SW','RP','RC','NW','FH','DM']
;          my_fields = ['ZZ','CZ','DR','RH','PH','KD','VR','SW','SD','RR','RP','RC','D0','NW','FH','N2','DM','MC','MP','MR','HC']
        endelse
;	if(site eq 'PHMO' or site eq 'PHKM' or site eq 'PHWA' or site eq 'PHKI') then begin
;        if(site eq 'KEVX' or site eq 'KMXX' or site eq 'KJGX') then begin
;	   my_fields = ['ZZ','CZ','DR','RH','PH','KD','VR','SW','RR','RP','RC','NW','FH','DM','HC','MC','MP','MR']
;	endif

;        rsl_radar_to_uf_gzip,radar,pol_uf_file,FIELDS=my_fields
;        rsl_radar_to_uf_gzip,radar,pol_uf_file
;        if (network eq 'GPMVN' or network eq 'GPMVNNPOL') then begin 
;	    rsl_radar_to_cfradial,radar,pol_uf_file,catch=0,FIELDS=my_fields
;	endif else begin
	    rsl_radar_to_cfradial,radar,pol_cf_file,catch=0,/dat2d,FIELDS=my_fields,/UFFIELDS
;           rsl_radar_to_cfradial,radar,pol_cf_file,catch=0,FIELDS=my_fields,/UFFIELDS
;	endelse
;        if (network eq 'GPMVN' or network eq 'GPMVNNPOL') then begin
;            radar = rsl_anyformat_to_radar(pol_uf_file,/quiet,ERROR=error)
;            print,'Final UF:'
;            printf,unit,'Final UF:'
;            print,'  --> ' + pol_uf_file
;            printf,unit,'  --> ' + pol_uf_file
;        endif else begin
            radar = rsl_anyformat_to_radar(pol_cf_file,/quiet,ERROR=error)
            print,'Final cf:  '
            printf,unit,'Final cf:  '
            print,'  --> ' + pol_cf_file       
            printf,unit,'  --> ' + pol_cf_file
;        endelse

            print
            print,'Final fields:  ',radar.volume.h.field_type
            print,'Scan Type:  ',radar.h.scan_mode
            print
            printf,unit
            printf,unit,'Final fields:  ', radar.volume.h.field_type
            printf,unit,'Scan Type:  ',radar.h.scan_mode
            printf,unit

plot_uf:
;
; *** Plot the results
;
        scan_type = radar.h.scan_mode
        if(DO_GRAPHICS) then begin
            print,'Plotting results...'
            spawn,'mkdir -p ' + multi_out
            plot_multi,radar,site,year,month,day,multi_out,/use_DPQC,/plot_ppi,/postscript
;           c = './multi_loop.csh ' + multi_out + ' ' + network + ' ' + year + ' ' + the_date
;           spawn,c
        endif
;
; ***gzip .cf files
;
;     if ((network eq 'GPMVN') || (network eq 'GPMVNNPOL')) then begin
;       print
;     endif else begin
       c = 'gzip ' + out_dir + '*.cf'
       spawn,c
       print,'Gzip cf files'
       print,'--> ' + out_dir
       printf,unit,'Gzip cf files'
       printf,unit,'--> ' + out_dir
;     endelse

;
; *** Get rid of tmp UF files
;
next_file:
radar=0; clear IDL memory to avoid overload
    endfor                      ; Files
;
; *** Print beg/end for processing time estimate
;
    print
    print,'Removing tmp directory: ' + tmp_dir
    printf,unit
    printf,unit,'Removing tmp directory: ' + tmp_dir
    spawn,'rm -r ' + tmp_dir
next_site:
;
; *** File check
;
;    if ((network eq 'GPMVN') || (network eq 'GPMVNNPOL')) then begin
;        wc = out_dir + '*.uf.gz'
;    endif else begin
        wc = out_dir + '*.gz'
;    endelse

    ffiles = file_search(wc,COUNT=nb)
    if(nb eq 0) then begin
           printf,unit
           printf,unit,'**************************************************************************************'
           printf,unit,'No final files found for ' + site
           printf,unit,'**************************************************************************************'
           printf,unit
           printf,unit,'________________________________________________________________________________'
           printf,unit
           printf,unit
           print,'No final files found for ' + site
    endif
;next_site:


    na = na + nb
   endfor ;sites
;
; ***Create loops of plotted images
;
     if(DO_GRAPHICS) then begin
        c = './multi_loop.csh ' + multi_out + ' ' + network + ' ' + year + ' ' + the_date
        spawn,c
     endif

    print
    printf,unit
    print,'Total files processed:  ',nt
    printf,unit,'Total files processed:  ',nt
    print,'Total final files :  ',na
    printf,unit,'Total final cf files :  ',na

;
; *** Write start/stop times
;
    spawn,'date',end_proc_time
;    print,'BEG: ',beg_proc_time
;    print,'END: ',end_proc_time
    a1 = strsplit(beg_proc_time,' ',/extract)
    t1 = a1[3]
    h1 = strmid(t1,0,2)
    m1 = strmid(t1,3,2)
    s1 = strmid(t1,6,2)
    motd1 = h1*3600 + m1*60 + s1*1

    a2 = strsplit(end_proc_time,' ',/extract)
    t2 = a2[3]
    h2 = strmid(t2,0,2)
    m2 = strmid(t2,3,2)
    s2 = strmid(t2,6,2)
    motd2 = h2*3600 + m2*60 + s2*1

    dt = abs(motd2-motd1)/60.0
    print,'Processed in ' + strcompress(dt) + ' minutes.'
    print
    printf,unit,'Processed in ' + strcompress(dt) + ' minutes.'
    printf,unit
    
    close,unit
    free_lun,unit
    
    print,'Success! Done.'
    stop
    return
    end