Files not closing

This is the place for queries that don't fit in any of the other categories.

Files not closing

Postby lukeh » Wed Oct 23, 2013 10:01 am

Hello everyone,

I have a simple function which reads netcdf data, calculated means and plots stuff. It's shown below:
Code: Select all
def plot_vertprof_clim(latlon = [50.92,6.36], \
   mths = range(5,6), days = range(1,2), hrs = range(0,3,3)):

    """
    Inputs
    ---
   latlon      Latitude/Longtitude point of vertical profile.
         List of int/float of latitude and longtitude, ie latlon = [lat,lon].

   mths      Which months to plot.
         List of integers of month number 5 to 8.
         For all months, set mths = range(5,9)

   days      Which days to plot.
         List of integers of day number 1 to 31.
         For a full month, set days = range(1,32), even if there are less than 31 days in the month.

        hrs      Which hours to plot.
         List of integers of hour, 3 hr intervals.
         For all times, set hrs = range(0,24,3)
    """
    month=mths
    day=days
    hour=hrs

    #--- Create a list of dates to process ---#

    date_list=[]

    for mths_idx in xrange(len(mths)):
   int_mth = int(mths[mths_idx])
   for days_idx in xrange(len(days)):
       int_day = int(days[days_idx])
       for hours_idx in xrange(len(hrs)):
      int_hour = int(hrs[hours_idx])
      try:
          dt.datetime(2008,int_mth,int_day)
      except ValueError:
          pass
      else:
          date_list.append(dt.datetime(2008,int_mth,int_day,int_hour))

   #--- Open files and extract desired variables ---#

    master_temp=[]
    master_relh=[]
    master_dust1=[]
    master_dust2=[]
    master_dust3=[]
    master_dust4=[]
    master_dust5=[]

    xlat,xlon=my_tools.latlon_to_ij(latlon[0],latlon[1])
    hght=my_tools.get_dust_hghts_mid()[0][:,xlat,xlon]
    for dte in date_list:
   temp_data=my_tools.get_dust_temp(dte)[0]
   sphum_data=my_tools.get_dust_sphum(dte)[0]
   dust_data1=my_tools.get_dust(dte,'code244')
   dust_data2=my_tools.get_dust(dte,'code245')
   dust_data3=my_tools.get_dust(dte,'code246')
   dust_data4=my_tools.get_dust(dte,'code247')
   dust_data5=my_tools.get_dust(dte,'code248')
   
   relh=my_tools.calc_dust_RH(sphum_data[:,xlat,xlon],temp_data[:,xlat,xlon],hght)
   temp=temp_data[:,xlat,xlon]
   dust1=dust_data1[0][:,xlat,xlon]
   dust2=dust_data2[0][:,xlat,xlon]
   dust3=dust_data3[0][:,xlat,xlon]
   dust4=dust_data4[0][:,xlat,xlon]
   dust5=dust_data5[0][:,xlat,xlon]
   
   master_temp.append(temp)
   master_relh.append(relh)
   master_dust1.append(dust1)
   master_dust2.append(dust2)
   master_dust3.append(dust3)
   master_dust4.append(dust4)
   master_dust5.append(dust5)

    temp_list=[]
    relh_list=[]
    dust1_list=[]
    dust2_list=[]
    dust3_list=[]
    dust4_list=[]
    dust5_list=[]

    temp_list.append(numpy.mean([master_temp[i] for i in xrange(len(master_temp))],axis=0))
    relh_list.append(numpy.mean([master_relh[i] for i in xrange(len(master_temp))],axis=0))
    dust1_list.append(numpy.mean([master_dust1[i] for i in xrange(len(master_temp))],axis=0))
    dust2_list.append(numpy.mean([master_dust2[i] for i in xrange(len(master_temp))],axis=0))
    dust3_list.append(numpy.mean([master_dust3[i] for i in xrange(len(master_temp))],axis=0))
    dust4_list.append(numpy.mean([master_dust4[i] for i in xrange(len(master_temp))],axis=0))
    dust5_list.append(numpy.mean([master_dust5[i] for i in xrange(len(master_temp))],axis=0))
   
    temp_list.append(numpy.median([master_temp[i] for i in xrange(len(master_temp))],axis=0))
    relh_list.append(numpy.median([master_relh[i] for i in xrange(len(master_temp))],axis=0))
    dust1_list.append(numpy.median([master_dust1[i] for i in xrange(len(master_temp))],axis=0))
    dust2_list.append(numpy.median([master_dust2[i] for i in xrange(len(master_temp))],axis=0))
    dust3_list.append(numpy.median([master_dust3[i] for i in xrange(len(master_temp))],axis=0))
    dust4_list.append(numpy.median([master_dust4[i] for i in xrange(len(master_temp))],axis=0))
    dust5_list.append(numpy.median([master_dust5[i] for i in xrange(len(master_temp))],axis=0))
   
    temp_list.append(numpy.std([master_temp[i] for i in xrange(len(master_temp))],axis=0))
    relh_list.append(numpy.std([master_relh[i] for i in xrange(len(master_temp))],axis=0))
    dust1_list.append(numpy.std([master_dust1[i] for i in xrange(len(master_temp))],axis=0))
    dust2_list.append(numpy.std([master_dust2[i] for i in xrange(len(master_temp))],axis=0))
    dust3_list.append(numpy.std([master_dust3[i] for i in xrange(len(master_temp))],axis=0))
    dust4_list.append(numpy.std([master_dust4[i] for i in xrange(len(master_temp))],axis=0))
    dust5_list.append(numpy.std([master_dust5[i] for i in xrange(len(master_temp))],axis=0))

    #--- Do some plotting ---#

    labels=['Mean','Median','Standard Deviation']
    for ind in xrange(3):

   plt.rcParams['font.size']=12
   plt.rcParams['figure.figsize']=[20,12]

   fig=plt.figure()
   ax=fig.add_subplot(171)
   plt.plot(temp_list[ind],hght,'k')
   plt.xlabel('Temperature\n(K)')
   plt.ylabel('Height (m)')

   ax=fig.add_subplot(172)
   plt.plot(relh_list[ind],hght,'k')
   plt.xlabel('Relative\nHumidity (%)')
   plt.yticks([])
   plt.xticks([0,50,100])
   plt.xlim(0,110)

   ax=fig.add_subplot(173)
   plt.plot(dust1_list[ind],hght,'k')
   plt.xlabel('0.166 um\n(pt/cm^3)')
   plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

   ax=fig.add_subplot(174)
   plt.plot(dust2_list[ind],hght,'k')
   plt.title(str(dte))
   plt.xlabel('0.5011 um\n(pt/cm^3)')
   plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
   plt.yticks([])
   
   ax=fig.add_subplot(175)
   plt.plot(dust3_list[ind],hght,'k')
   plt.xlabel('1.5136 um\n(pt/cm^3)')
   plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

   ax=fig.add_subplot(176)
   plt.plot(dust4_list[ind],hght,'k')
   plt.xlabel('4.5709 um\n(pt/cm^3)')
   plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

   ax=fig.add_subplot(177)
   plt.plot(dust5_list[ind],hght,'k')
   plt.xlabel('13.804 um\n(pt/cm^3)')
   plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
   
   plt.show()

    return


It uses several other functions (in my_tools.py) to extract the variables from the netcdf files. An example of one is shown below:

Code: Select all
def get_dust_temp(date):

    """
    INPUT:   date   Date of file as a datetime object.

    OUTPUT:   temp   Model Temperatures for that date, in Kelivn.
      lat   Latitude (degrees East)
      lon   Longtitude (degrees North)

    """
   
    yr = date.year
    mth = date.month
    day = date.day
    hr = date.hour

    file=sinet.netcdf_file('/path to netcdf file.nc','r')

    var = file.variables

    time = var['time'][:]   # hours since 2008-05-01 00:00:00
    level = var['lev'][:]   # level: 1 model top, 40 surface
    lat = var['lat'][:]      # degrees North
    lon = var['lon'][:]      # degrees East
    temp = var['var11'][:]   # Temp (K): temp[time][level][lat][lon]

    file.close()

    return temp[0], lat, lon


It crashes with error: [Errno 24] Too many open files. This is despite the file.close() in get_dust_temp.
There are 6 times and 7 files per day for 31 days, so it runs to completion if i do ulimit -n 1800, but this doesn't solve the problem of not closing the files.
Any ideas of what's going on?
Thanks in advance,
Last edited by Mekire on Wed Oct 23, 2013 11:51 am, edited 1 time in total.
Reason: First post lock.
lukeh
 
Posts: 4
Joined: Wed Oct 23, 2013 9:42 am

Re: Files not closing

Postby lukeh » Wed Oct 23, 2013 11:25 am

I just realised the indentations on the first code segment were not correctly copied. It should look more like this:

Code: Select all
    def plot_vertprof_clim(latlon = [50.92,6.36], \
       mths = range(5,6), days = range(1,2), hrs = range(0,3,3)):

        """
        Inputs
        ---
       latlon      Latitude/Longtitude point of vertical profile.
             List of int/float of latitude and longtitude, ie latlon = [lat,lon].

       mths      Which months to plot.
             List of integers of month number 5 to 8.
             For all months, set mths = range(5,9)

       days      Which days to plot.
             List of integers of day number 1 to 31.
             For a full month, set days = range(1,32), even if there are less than 31 days in the month.

            hrs      Which hours to plot.
             List of integers of hour, 3 hr intervals.
             For all times, set hrs = range(0,24,3)
        """
        month=mths
        day=days
        hour=hrs

        #--- Create a list of dates to process ---#

        date_list=[]

        for mths_idx in xrange(len(mths)):
           int_mth = int(mths[mths_idx])
           for days_idx in xrange(len(days)):
               int_day = int(days[days_idx])
               for hours_idx in xrange(len(hrs)):
                  int_hour = int(hrs[hours_idx])
                      try:
                          dt.datetime(2008,int_mth,int_day)
                      except ValueError:
                          pass
                      else:
                          date_list.append(dt.datetime(2008,int_mth,int_day,int_hour))

       #--- Open files and extract desired variables ---#

        master_temp=[]
        master_relh=[]
        master_dust1=[]
        master_dust2=[]
        master_dust3=[]
        master_dust4=[]
        master_dust5=[]

        xlat,xlon=my_tools.latlon_to_ij(latlon[0],latlon[1])
        hght=my_tools.get_dust_hghts_mid()[0][:,xlat,xlon]
        for dte in date_list:
            temp_data=my_tools.get_dust_temp(dte)[0]
            sphum_data=my_tools.get_dust_sphum(dte)[0]
            dust_data1=my_tools.get_dust(dte,'code244')
            dust_data2=my_tools.get_dust(dte,'code245')
            dust_data3=my_tools.get_dust(dte,'code246')
            dust_data4=my_tools.get_dust(dte,'code247')
            dust_data5=my_tools.get_dust(dte,'code248')
       
           relh=my_tools.calc_dust_RH(sphum_data[:,xlat,xlon],temp_data[:,xlat,xlon],hght)
           temp=temp_data[:,xlat,xlon]
           dust1=dust_data1[0][:,xlat,xlon]
           dust2=dust_data2[0][:,xlat,xlon]
           dust3=dust_data3[0][:,xlat,xlon]
           dust4=dust_data4[0][:,xlat,xlon]
           dust5=dust_data5[0][:,xlat,xlon]
       
           master_temp.append(temp)
           master_relh.append(relh)
           master_dust1.append(dust1)
           master_dust2.append(dust2)
           master_dust3.append(dust3)
           master_dust4.append(dust4)
           master_dust5.append(dust5)

       temp_list=[]
       relh_list=[]
       dust1_list=[]
       dust2_list=[]
       dust3_list=[]
       dust4_list=[]
       dust5_list=[]

       temp_list.append(numpy.mean([master_temp[i] for i in xrange(len(master_temp))],axis=0))
       relh_list.append(numpy.mean([master_relh[i] for i in xrange(len(master_temp))],axis=0))
       dust1_list.append(numpy.mean([master_dust1[i] for i in xrange(len(master_temp))],axis=0))
       dust2_list.append(numpy.mean([master_dust2[i] for i in xrange(len(master_temp))],axis=0))
       dust3_list.append(numpy.mean([master_dust3[i] for i in xrange(len(master_temp))],axis=0))
       dust4_list.append(numpy.mean([master_dust4[i] for i in xrange(len(master_temp))],axis=0))
       dust5_list.append(numpy.mean([master_dust5[i] for i in xrange(len(master_temp))],axis=0))
       
       temp_list.append(numpy.median([master_temp[i] for i in xrange(len(master_temp))],axis=0))
       relh_list.append(numpy.median([master_relh[i] for i in xrange(len(master_temp))],axis=0))
       dust1_list.append(numpy.median([master_dust1[i] for i in xrange(len(master_temp))],axis=0))
       dust2_list.append(numpy.median([master_dust2[i] for i in xrange(len(master_temp))],axis=0))
       dust3_list.append(numpy.median([master_dust3[i] for i in xrange(len(master_temp))],axis=0))
       dust4_list.append(numpy.median([master_dust4[i] for i in xrange(len(master_temp))],axis=0))
       dust5_list.append(numpy.median([master_dust5[i] for i in xrange(len(master_temp))],axis=0))
       
       temp_list.append(numpy.std([master_temp[i] for i in xrange(len(master_temp))],axis=0))
       relh_list.append(numpy.std([master_relh[i] for i in xrange(len(master_temp))],axis=0))
       dust1_list.append(numpy.std([master_dust1[i] for i in xrange(len(master_temp))],axis=0))
       dust2_list.append(numpy.std([master_dust2[i] for i in xrange(len(master_temp))],axis=0))
       dust3_list.append(numpy.std([master_dust3[i] for i in xrange(len(master_temp))],axis=0))
       dust4_list.append(numpy.std([master_dust4[i] for i in xrange(len(master_temp))],axis=0))
       dust5_list.append(numpy.std([master_dust5[i] for i in xrange(len(master_temp))],axis=0))

        #--- Do some plotting ---#

        labels=['Mean','Median','Standard Deviation']
        for ind in xrange(3):

           plt.rcParams['font.size']=12
           plt.rcParams['figure.figsize']=[20,12]

           fig=plt.figure()
           ax=fig.add_subplot(171)
           plt.plot(temp_list[ind],hght,'k')
           plt.xlabel('Temperature\n(K)')
           plt.ylabel('Height (m)')

           ax=fig.add_subplot(172)
           plt.plot(relh_list[ind],hght,'k')
           plt.xlabel('Relative\nHumidity (%)')
           plt.yticks([])
           plt.xticks([0,50,100])
           plt.xlim(0,110)

           ax=fig.add_subplot(173)
           plt.plot(dust1_list[ind],hght,'k')
           plt.xlabel('0.166 um\n(pt/cm^3)')
           plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

           ax=fig.add_subplot(174)
           plt.plot(dust2_list[ind],hght,'k')
           plt.title(str(dte))
           plt.xlabel('0.5011 um\n(pt/cm^3)')
           plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
           plt.yticks([])
       
           ax=fig.add_subplot(175)
           plt.plot(dust3_list[ind],hght,'k')
           plt.xlabel('1.5136 um\n(pt/cm^3)')
           plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

           ax=fig.add_subplot(176)
           plt.plot(dust4_list[ind],hght,'k')
           plt.xlabel('4.5709 um\n(pt/cm^3)')
           plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))

           ax=fig.add_subplot(177)
           plt.plot(dust5_list[ind],hght,'k')
           plt.xlabel('13.804 um\n(pt/cm^3)')
           plt.ticklabel_format(style='sci', axis='x', scilimits=(0,0))
       
           plt.show()

        return

lukeh
 
Posts: 4
Joined: Wed Oct 23, 2013 9:42 am

Re: Files not closing

Postby ochichinyezaboombwa » Wed Oct 23, 2013 9:25 pm

1st, it's best if you provide the full traceback (that includes important information like on which line it crashes) instead of just saying "It crashes".
2nd, I have no idea what is sinet.netcdf_file and what does it create and whether the build-in close() should work on this object (maybe there is something like sinet.close()?). To see what actually happens (whether your files are indeed closed), add a couple of print statements, e.g.:
Code: Select all
>>> fp = open("aaa", "w")
>>> fp
<open file 'aaa', mode 'w' at 0xb74af078>
>>> fp.close()
>>> fp
<closed file 'aaa', mode 'w' at 0xb74af078>
ochichinyezaboombwa
 
Posts: 200
Joined: Tue Jun 04, 2013 7:53 pm

Re: Files not closing

Postby lukeh » Thu Oct 24, 2013 10:04 am

Thanks for the reply. I think the error is the file.close() is not actually closing the file in scipy.io.netcdf:

Code: Select all
>>> import scipy.io.netcdf as sinet
>>> fid=sinet.netcdf_file('netcdf file path','r')
>>> fid
<scipy.io.netcdf.netcdf_file at 0x241f890>
>>> fid.variables['lev'][:]
array([  1.,   2.,   3.,   4.,   5.,   6.,   7.,   8.,   9.,  10.,  11.,
        12.,  13.,  14.,  15.,  16.,  17.,  18.,  19.,  20.,  21.,  22.,
        23.,  24.,  25.,  26.,  27.,  28.,  29.,  30.,  31.,  32.,  33.,
        34.,  35.,  36.,  37.,  38.,  39.,  40.])
>>> fid.close()
>>> fid
<scipy.io.netcdf.netcdf_file at 0x241f890>
>>> fid.variables['lev'][:]
array([  1.,   2.,   3.,   4.,   5.,   6.,   7.,   8.,   9.,  10.,  11.,
        12.,  13.,  14.,  15.,  16.,  17.,  18.,  19.,  20.,  21.,  22.,
        23.,  24.,  25.,  26.,  27.,  28.,  29.,  30.,  31.,  32.,  33.,
        34.,  35.,  36.,  37.,  38.,  39.,  40.])


It doesn't explicitly say the file is open after the fid.close(), but I can still access the 'lev' variable. There doesn't seem to be another built in method to close the files. Any suggestions?
Thanks.
lukeh
 
Posts: 4
Joined: Wed Oct 23, 2013 9:42 am

Re: Files not closing

Postby stranac » Thu Oct 24, 2013 11:39 am

Maybe using it as a context manager would help:
Code: Select all
with sinet.netcdf_file('netcdf file path', 'r') as fid:
    # do whatever

That's just a guess, though.
Friendship is magic!

R.I.P. Tracy M. You will be missed.
User avatar
stranac
 
Posts: 1151
Joined: Thu Feb 07, 2013 3:42 pm

Re: Files not closing

Postby lukeh » Thu Oct 24, 2013 11:59 am

Thanks for all the help and suggestions. I think I'm just going to use Scientific Python to handle netcdf files:

http://dirac.cnrs-orleans.fr/plone/soft ... ficpython/

I've installed it and done a test on one file, and the file.close() method works.

So I guess we can consider this file closed...
lukeh
 
Posts: 4
Joined: Wed Oct 23, 2013 9:42 am


Return to General Coding Help

Who is online

Users browsing this forum: Baidu [Spider], pete1845 and 1 guest