How should I specify cookies for an HTTP protocol open request?

[jthielen]

I am looking to create a Pangeo Forge recipe for data hosted on NCAR RDA, which requires a session login to access the files via HTTP. However, I am failing to execute said recipe due to receiving a 403 Forbidden error on fsspec's attempt to open the HTTP file (which arises during a self.info call, which seems weird?), and so, I was hoping to receive advice on how to specify the requests.cookies.RequestsCookieJar to avoid this error (or somewhere else I am going wrong)!

A minimal reproducible example comparing direct usage of requests.get versus fsspec.open is below (note this requires an account to actually use):

import fsspec
import getpass
import requests

# Get a login session to have proper cookies
username = input("Enter RDA Username: ")
userpass = getpass.getpass("Enter RDA Password: ")

url = 'https://rda.ucar.edu/cgi-bin/login'
values = {'email': username, 'passwd': userpass, 'action': 'login'}
# Authenticate
login = requests.post(url, data=values)
if login.status_code != 200:
    print('Bad Authentication')
    print(login.text)
else:
    requests_kwargs = {'cookies': login.cookies, 'allow_redirects': False}

# Configuration
filename = 'https://rda.ucar.edu/data/OS/ds841.6/volumes/2010/20100120/nexrad_3d_v4_2_20100120T180500Z.nc'
file_base = filename.split('/')[-1]
chunk_size=1048576

This works:

req = requests.get(filename, stream=True, **requests_kwargs)
with open(file_base, 'wb') as outfile:
    for chunk in req.iter_content(chunk_size=chunk_size):
        outfile.write(chunk)

This fails

remote_file = fsspec.open(filename, **requests_kwargs)
with remote_file as infile:
    with open(file_base, 'wb') as outfile:
        while chunk := infile.read(chunk_size):
            outfile.write(chunk)

---------------------------------------------------------------------------
ClientResponseError                       Traceback (most recent call last)
File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/fsspec/implementations/http.py:383, in HTTPFileSystem._info(self, url, **kwargs)
    381 try:
    382     info.update(
--> 383         await _file_info(
    384             url,
    385             size_policy=policy,
    386             session=session,
    387             **self.kwargs,
    388             **kwargs,
    389         )
    390     )
    391     if info.get("size") is not None:

File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/fsspec/implementations/http.py:734, in _file_info(url, session, size_policy, **kwargs)
    733 async with r:
--> 734     r.raise_for_status()
    736     # TODO:
    737     #  recognise lack of 'Accept-Ranges',
    738     #                 or 'Accept-Ranges': 'none' (not 'bytes')
    739     #  to mean streaming only, no random access => return None

File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/aiohttp/client_reqrep.py:1004, in ClientResponse.raise_for_status(self)
   1003 self.release()
-> 1004 raise ClientResponseError(
   1005     self.request_info,
   1006     self.history,
   1007     status=self.status,
   1008     message=self.reason,
   1009     headers=self.headers,
   1010 )

ClientResponseError: 403, message='Forbidden', url=URL('https://rda.ucar.edu/data/OS/ds841.6/volumes/2010/20100120/nexrad_3d_v4_2_20100120T180500Z.nc')

The above exception was the direct cause of the following exception:

FileNotFoundError                         Traceback (most recent call last)
Input In [7], in <cell line: 3>()
      1 # This raises a 403 unauthorized error
      2 remote_file = fsspec.open(filename, **requests_kwargs)
----> 3 with remote_file as infile:
      4     with open(file_base, 'wb') as outfile:
      5         while chunk := infile.read(chunk_size):

File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/fsspec/core.py:103, in OpenFile.__enter__(self)
    100 def __enter__(self):
    101     mode = self.mode.replace("t", "").replace("b", "") + "b"
--> 103     f = self.fs.open(self.path, mode=mode)
    105     self.fobjects = [f]
    107     if self.compression is not None:

File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/fsspec/spec.py:1009, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
   1007 else:
   1008     ac = kwargs.pop("autocommit", not self._intrans)
-> 1009     f = self._open(
   1010         path,
   1011         mode=mode,
   1012         block_size=block_size,
   1013         autocommit=ac,
   1014         cache_options=cache_options,
   1015         **kwargs,
   1016     )
   1017     if compression is not None:
   1018         from fsspec.compression import compr

File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/fsspec/implementations/http.py:343, in HTTPFileSystem._open(self, path, mode, block_size, autocommit, cache_type, cache_options, size, **kwargs)
    341 kw["asynchronous"] = self.asynchronous
    342 kw.update(kwargs)
--> 343 size = size or self.info(path, **kwargs)["size"]
    344 session = sync(self.loop, self.set_session)
    345 if block_size and size:

File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/fsspec/asyn.py:85, in sync_wrapper.<locals>.wrapper(*args, **kwargs)
     82 @functools.wraps(func)
     83 def wrapper(*args, **kwargs):
     84     self = obj or args[0]
---> 85     return sync(self.loop, func, *args, **kwargs)

File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/fsspec/asyn.py:65, in sync(loop, func, timeout, *args, **kwargs)
     63     raise FSTimeoutError from return_result
     64 elif isinstance(return_result, BaseException):
---> 65     raise return_result
     66 else:
     67     return return_result

File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/fsspec/asyn.py:25, in _runner(event, coro, result, timeout)
     23     coro = asyncio.wait_for(coro, timeout=timeout)
     24 try:
---> 25     result[0] = await coro
     26 except Exception as ex:
     27     result[0] = ex

File ~/miniconda3/envs/pangeo-forge/lib/python3.10/site-packages/fsspec/implementations/http.py:396, in HTTPFileSystem._info(self, url, **kwargs)
    393     except Exception as exc:
    394         if policy == "get":
    395             # If get failed, then raise a FileNotFoundError
--> 396             raise FileNotFoundError(url) from exc
    397         logger.debug(str(exc))
    399 return {"name": url, "size": None, **info, "type": "file"}

FileNotFoundError: https://rda.ucar.edu/data/OS/ds841.6/volumes/2010/20100120/nexrad_3d_v4_2_20100120T180500Z.nc

[cisaacstern]

@jthielen, out of curiosity what happens if you pass username and password directly to fsspec.open as in

kwargs = dict(username=username, password=userpass)
remote_file = fsspec.open(filename, **kwargs)

?

[jthielen]

This was indeed one of the first things I tried, but it (and any other access method oriented around direct HTTP authentication) unfortunately still gives 403 Forbidden. Only methods that have worked require first logging in to the specified endpoint, and then supplying the login session cookie on further requests.

[martindurant]

You need to include your cookies in client_kwargs:

fsspec.open(filename, client_kwargs=requests_kwargs)

Note that fsspec's HTTP backend uses aiohttp, not requests.

[jthielen]

This was the key! I read this line: "storage_options: key-value Any other parameters passed on to requests" and mistakenly assumed requests the package was what the backend used...while being totally oblivious to the "client_kwargs: dict Passed to aiohttp.ClientSession" line right above it.

Here is the successful code snippet:

username = input("Enter RDA Username: ")
userpass = getpass.getpass("Enter RDA Password: ")

url = 'https://rda.ucar.edu/cgi-bin/login'
values = {'email': username, 'passwd': userpass, 'action': 'login'}

async with aiohttp.ClientSession() as session:
    async with session.post(url, data=values) as response:
        cookies = response.cookies
aiohttp_kwargs = {'cookies': cookies}

# Configuration
filename = 'https://rda.ucar.edu/data/OS/ds841.6/volumes/2010/20100120/nexrad_3d_v4_2_20100120T180500Z.nc'
file_base = filename.split('/')[-1]
chunk_size=1048576

remote_file = fsspec.open(filename, client_kwargs=aiohttp_kwargs)
with remote_file as infile:
    with open(file_base, 'wb') as outfile:
        while chunk := infile.read(chunk_size):
            outfile.write(chunk)

[martindurant]

Ha, "requests" as opposed to requests. Perhaps the doc could be clearer.