resource not closed on exception for non-existant file

[jreback]

from pandas-dev/pandas#17058

In [7]: import s3fs

In [8]: s3fs.__version__
Out[8]: '0.1.1'

In [5]: fs = s3fs.S3FileSystem(anon=True)

In [6]: fs.open('nyqpug/asdf.csv')
---------------------------------------------------------------------------
ClientError                               Traceback (most recent call last)
/Users/jreback/miniconda3/envs/pandas/lib/python3.6/site-packages/s3fs/core.py in info(self, path, refresh, **kwargs)
    393                 out = self._call_s3(self.s3.head_object,
--> 394                                     kwargs, Bucket=bucket, Key=key, **self.req_kw)
    395                 out = {'ETag': out['ETag'], 'Key': '/'.join([bucket, key]),

/Users/jreback/miniconda3/envs/pandas/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
    167         additional_kwargs.update(kwargs)
--> 168         return method(**additional_kwargs)
    169 

/Users/jreback/miniconda3/envs/pandas/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
    252             # The "self" in this scope is referring to the BaseClient.
--> 253             return self._make_api_call(operation_name, kwargs)
    254 

/Users/jreback/miniconda3/envs/pandas/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
    543             error_class = self.exceptions.from_code(error_code)
--> 544             raise error_class(parsed_response, operation_name)
    545         else:

ClientError: An error occurred (404) when calling the HeadObject operation: Not Found

During handling of the above exception, another exception occurred:

FileNotFoundError                         Traceback (most recent call last)
<ipython-input-6-7628c3834aad> in <module>()
----> 1 fs.open('nyqpug/asdf.csv')

/Users/jreback/miniconda3/envs/pandas/lib/python3.6/site-packages/s3fs/core.py in open(self, path, mode, block_size, acl, fill_cache, **kwargs)
    289                                       " and manage bytes" % (mode[0] + 'b'))
    290         return S3File(self, path, mode, block_size=block_size, acl=acl,
--> 291                       fill_cache=fill_cache, s3_additional_kwargs=kwargs)
    292 
    293     def _lsdir(self, path, refresh=False):

/Users/jreback/miniconda3/envs/pandas/lib/python3.6/site-packages/s3fs/core.py in __init__(self, s3, path, mode, block_size, acl, fill_cache, s3_additional_kwargs)
    927         else:
    928             try:
--> 929                 self.size = self.info()['Size']
    930             except (ClientError, ParamValidationError):
    931                 raise IOError("File not accessible", path)

/Users/jreback/miniconda3/envs/pandas/lib/python3.6/site-packages/s3fs/core.py in info(self, **kwargs)
    936     def info(self, **kwargs):
    937         """ File information about this path """
--> 938         return self.s3.info(self.path, **kwargs)
    939 
    940     def metadata(self, refresh=False, **kwargs):

/Users/jreback/miniconda3/envs/pandas/lib/python3.6/site-packages/s3fs/core.py in info(self, path, refresh, **kwargs)
    398                 return out
    399             except (ClientError, ParamValidationError):
--> 400                 raise FileNotFoundError(path)
    401 
    402     _metadata_cache = {}

FileNotFoundError: nyqpug/asdf.csv

This shows a resource not closed warning on python 3.

[martindurant]

Sorry, I don't see a warning - I get the same (expected) traceback as above on py3. I forget the syntax, how do you get rid of the upper portion and the "during handling" part?

[jreback]

hmm, maybe its just how its used in pandas code, I can't seem to repro the actual warning except by running out test suite. I suspect some global state is being modified.

[jreback]

boto/boto3#454 unresolved bug

closing as out of our hands.