Error when running gpt-j-online.ipynb example: AttributeError: 'JobConfig' object has no attribute 'py_driver_sys_path'

[AlexBulankou]

Following the steps to access JupyterLab, I install ray from terminal with pip install -U "ray[air]" and then run gpt-j-online example. I get the following error:

ConnectionAbortedError: Initialization failure from server:
Traceback (most recent call last):
  File "/home/ray/anaconda3/lib/python3.10/site-packages/ray/util/client/server/proxier.py", line 703, in Datapath
    if not self.proxy_manager.start_specific_server(
  File "/home/ray/anaconda3/lib/python3.10/site-packages/ray/util/client/server/proxier.py", line 298, in start_specific_server
    runtime_env_config = job_config.get_proto_runtime_env_config()
  File "/home/ray/anaconda3/lib/python3.10/site-packages/ray/job_config.py", line 143, in get_proto_runtime_env_config
    return self.get_proto_job_config().runtime_env_info.runtime_env_config
  File "/home/ray/anaconda3/lib/python3.10/site-packages/ray/job_config.py", line 115, in get_proto_job_config
    pb.py_driver_sys_path.extend(self.py_driver_sys_path)
AttributeError: 'JobConfig' object has no attribute 'py_driver_sys_path'

[richardsliu]

Try using pip install ray==2.3 and then pip install ray[serve].

[AlexBulankou]

I did as you suggested:

pip install ray==2.3 
pip install ray[serve]

Then I hit another issue: `AttributeError: module 'pydantic.fields' has no attribute. I found this issue that seems related and tried to add "pydantic<2", but I still hit this issue:

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[5], line 1
----> 1 ray.init(
      2     address="ray://example-cluster-kuberay-head-svc:10001",
      3     runtime_env={
      4         "pip": [
      5             "IPython",
      6             "boto3==1.26",
      7             "botocore==1.29", 
      8             "datasets",
      9             "fastapi",
     10             "accelerate>=0.16.0",
     11             "transformers>=4.26.0",
     12             "numpy<1.24",  # remove when mlflow updates beyond 2.2
     13             "torch",
     14             "pydantic<2",
     15         ]
     16     }
     17 )

File /opt/conda/lib/python3.10/site-packages/ray/_private/client_mode_hook.py:105, in client_mode_hook.<locals>.wrapper(*args, **kwargs)
    103     if func.__name__ != "init" or is_client_mode_enabled_by_default:
    104         return getattr(ray, func.__name__)(*args, **kwargs)
--> 105 return func(*args, **kwargs)

File /opt/conda/lib/python3.10/site-packages/ray/_private/worker.py:1267, in init(address, num_cpus, num_gpus, resources, object_store_memory, local_mode, ignore_reinit_error, include_dashboard, dashboard_host, dashboard_port, job_config, configure_logging, logging_level, logging_format, log_to_driver, namespace, runtime_env, storage, **kwargs)
   1265 passed_kwargs.update(kwargs)
   1266 builder._init_args(**passed_kwargs)
-> 1267 ctx = builder.connect()
   1268 from ray._private.usage import usage_lib
   1270 if passed_kwargs.get("allow_multiple") is True:

File /opt/conda/lib/python3.10/site-packages/ray/client_builder.py:182, in ClientBuilder.connect(self)
    179 if self._allow_multiple_connections:
    180     old_ray_cxt = ray.util.client.ray.set_context(None)
--> 182 client_info_dict = ray.util.client_connect.connect(
    183     self.address,
    184     job_config=self._job_config,
    185     _credentials=self._credentials,
    186     ray_init_kwargs=self._remote_init_kwargs,
    187     metadata=self._metadata,
    188 )
    190 dashboard_url = ray.util.client.ray._get_dashboard_url()
    192 cxt = ClientContext(
    193     dashboard_url=dashboard_url,
    194     python_version=client_info_dict["python_version"],
   (...)
    199     _context_to_restore=ray.util.client.ray.get_context(),
    200 )

File /opt/conda/lib/python3.10/site-packages/ray/util/client_connect.py:57, in connect(conn_str, secure, metadata, connection_retries, job_config, namespace, ignore_version, _credentials, ray_init_kwargs)
     52 _explicitly_enable_client_mode()
     54 # TODO(barakmich): https://github.com/ray-project/ray/issues/13274
     55 # for supporting things like cert_path, ca_path, etc and creating
     56 # the correct metadata
---> 57 conn = ray.connect(
     58     conn_str,
     59     job_config=job_config,
     60     secure=secure,
     61     metadata=metadata,
     62     connection_retries=connection_retries,
     63     namespace=namespace,
     64     ignore_version=ignore_version,
     65     _credentials=_credentials,
     66     ray_init_kwargs=ray_init_kwargs,
     67 )
     68 return conn

File /opt/conda/lib/python3.10/site-packages/ray/util/client/__init__.py:252, in RayAPIStub.connect(self, *args, **kw_args)
    250 def connect(self, *args, **kw_args):
    251     self.get_context()._inside_client_test = self._inside_client_test
--> 252     conn = self.get_context().connect(*args, **kw_args)
    253     global _lock, _all_contexts
    254     with _lock:

File /opt/conda/lib/python3.10/site-packages/ray/util/client/__init__.py:105, in _ClientContext.connect(self, conn_str, job_config, secure, metadata, connection_retries, namespace, ignore_version, _credentials, ray_init_kwargs)
    103     conn_info = self.client_worker.connection_info()
    104     self._check_versions(conn_info, ignore_version)
--> 105     self._register_serializers()
    106     return conn_info
    107 except Exception:

File /opt/conda/lib/python3.10/site-packages/ray/util/client/__init__.py:121, in _ClientContext._register_serializers(self)
    118 from ray.util.serialization import StandaloneSerializationContext
    120 ctx = StandaloneSerializationContext()
--> 121 ray.util.serialization_addons.apply(ctx)

File /opt/conda/lib/python3.10/site-packages/ray/util/serialization_addons.py:58, in apply(serialization_context)
     56 @DeveloperAPI
     57 def apply(serialization_context):
---> 58     register_pydantic_serializer(serialization_context)
     59     register_starlette_serializer(serialization_context)
     61     if sys.platform != "win32":

File /opt/conda/lib/python3.10/site-packages/ray/util/serialization_addons.py:21, in register_pydantic_serializer(serialization_context)
     16     return
     18 # Pydantic's Cython validators are not serializable.
     19 # https://github.com/cloudpipe/cloudpickle/issues/408
     20 serialization_context._register_cloudpickle_serializer(
---> 21     pydantic.fields.ModelField,
     22     custom_serializer=lambda o: {
     23         "name": o.name,
     24         # outer_type_ is the original type for ModelFields,
     25         # while type_ can be updated later with the nested type
     26         # like int for List[int].
     27         "type_": o.outer_type_,
     28         "class_validators": o.class_validators,
     29         "model_config": o.model_config,
     30         "default": o.default,
     31         "default_factory": o.default_factory,
     32         "required": o.required,
     33         "alias": o.alias,
     34         "field_info": o.field_info,
     35     },
     36     custom_deserializer=lambda kwargs: pydantic.fields.ModelField(**kwargs),
     37 )

AttributeError: module 'pydantic.fields' has no attribute 'ModelField'

[AlexBulankou]

The workaround here was to run:

pip install "pydantic<2"

and that addressed the problem.