bugfix: evaluation thread timeout

[NIL-zhuang]

The current version of evaluation contains a threading timeout issue refered in issue #854 #722 #734 #810, which makes the evaluation process hangs in the last few cases and won't finish running.

This PR adds a threading_timeout parameter in the run_config, which raises asyncio.TimeoutError when the metric ascore function runs over the threading_timeout, avoiding the forever hanging situation.

[jjmachan]

hey @NIL-zhuang thank you so much for putting in this PR and my sincere apologies for not merging this sooner 🙏🏽

[kmr666]

The current version of evaluation contains a threading timeout issue refered in issue #854 #722 #734 #810, which makes the evaluation process hangs in the last few cases and won't finish running.

This PR adds a threading_timeout parameter in the run_config, which raises asyncio.TimeoutError when the metric ascore function runs over the threading_timeout, avoiding the forever hanging situation.

Hello, I am encountering an error when running the single metric 'context_relevancy' with version 0.1.10 of Ragas. The specific error message is as follows:
Evaluating: 0%| | 0/100 [00:03<?, ?it/s]
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Program Files\Python311\Lib\site-packages\httpx_transports\default.py", line 69, in map_httpcore_exceptions
yield
File "C:\Program Files\Python311\Lib\site-packages\httpx_transports\default.py", line 373, in handle_async_request
resp = await self._pool.handle_async_request(req)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\httpcore_async\connection_pool.py", line 216, in handle_async_request
raise exc from None
File "C:\Program Files\Python311\Lib\site-packages\httpcore_async\connection_pool.py", line 196, in handle_async_request
response = await connection.handle_async_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\httpcore_async\http_proxy.py", line 317, in handle_async_request
stream = await stream.start_tls(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\httpcore_async\http11.py", line 383,
in start_tls
return await self._stream.start_tls(ssl_context, server_hostname, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\httpcore_backends\anyio.py", line 68, in start_tls
with map_exceptions(exc_map):
File "C:\Program Files\Python311\Lib\contextlib.py", line 158, in exit
self.gen.throw(typ, value, traceback)
File "C:\Program Files\Python311\Lib\site-packages\httpcore_exceptions.py", line 14, in
map_exceptions
raise to_exc(exc) from exc
httpcore.ConnectError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "C:\Program Files\Python311\Lib\site-packages\openai_base_client.py", line 1548, in _request
response = await self._client.send(
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\httpx_client.py", line 1661, in send
response = await self._send_handling_auth(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\httpx_client.py", line 1689, in _send_handling_auth
response = await self._send_handling_redirects(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\httpx_client.py", line 1726, in _send_handling_redirects
response = await self._send_single_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\httpx_client.py", line 1763, in _send_single_request
response = await transport.handle_async_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\httpx_transports\default.py", line 372, in handle_async_request
with map_httpcore_exceptions():
File "C:\Program Files\Python311\Lib\contextlib.py", line 158, in exit
self.gen.throw(typ, value, traceback)
File "C:\Program Files\Python311\Lib\site-packages\httpx_transports\default.py", line 86, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "C:\Program Files\Python311\Lib\threading.py", line 1045, in _bootstrap_inner
self.run()
File "C:\Program Files\Python311\Lib\site-packages\ragas\executor.py", line 87, in run
results = self.loop.run_until_complete(self.aresults())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\asyncio\base_events.py", line 654, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\ragas\executor.py", line 79, in aresults
r = await future
^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\asyncio\tasks.py", line 615, in wait_for_one
return f.result() # May raise f.exception().
^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\ragas\executor.py", line 37, in sema_coro
return await coro
^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\ragas\executor.py", line 109, in wrapped_callable_async
raise e
File "C:\Program Files\Python311\Lib\site-packages\ragas\executor.py", line 104, in wrapped_callable_async
result = await callable(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\ragas\metrics\base.py", line 134, in ascore
raise e
File "C:\Program Files\Python311\Lib\site-packages\ragas\metrics\base.py", line 127, in ascore
score = await asyncio.wait_for(
^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\asyncio\tasks.py", line 489, in wait_for
return fut.result()
^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\ragas\metrics_context_relevancy.py",
line 77, in ascore
result = await self.llm.generate(
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\ragas\llms\base.py", line 93, in generate
return await agenerate_text_with_retry(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\tenacity\asyncio_init.py", line 189, in async_wrapped
return await copy(fn, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\tenacity\asyncio_init.py", line 111, in call
do = await self.iter(retry_state=retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\tenacity\asyncio_init.py", line 153, in iter
result = await action(retry_state)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\tenacity_utils.py", line 99, in inner return call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\tenacity_init.py", line 398, in
self._add_action_func(lambda rs: rs.outcome.result())
^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\concurrent\futures_base.py", line 449, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\concurrent\futures_base.py", line 401, in __get_result
raise self.exception
File "C:\Program Files\Python311\Lib\site-packages\tenacity\asyncio_init.py", line 114, in call
result = await fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\ragas\llms\base.py", line 170, in agenerate_text
return await self.langchain_llm.agenerate_prompt(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain_core\language_models\chat_models.py", line 691, in agenerate_prompt
return await self.agenerate(
^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain_core\language_models\chat_models.py", line 651, in agenerate
raise exceptions[0]
File "C:\Program Files\Python311\Lib\site-packages\langchain_core\language_models\chat_models.py", line 836, in _agenerate_with_cache
result = await self._agenerate(
^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\langchain_openai\chat_models\base.py", line 674, in _agenerate
response = await self.async_client.create(**payload)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\openai\resources\chat\completions.py", line 1289, in create
return await self._post(
^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\openai_base_client.py", line 1816, in post
return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\openai_base_client.py", line 1514, in request
return await self._request(
^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\openai_base_client.py", line 1572, in _request
return await self._retry_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\openai_base_client.py", line 1641, in _retry_request
return await self._request(
^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\openai_base_client.py", line 1572, in _request
return await self._retry_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\openai_base_client.py", line 1641, in _retry_request
return await self._request(
^^^^^^^^^^^^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\openai_base_client.py", line 1582, in _request
raise APIConnectionError(request=request) from err
openai.APIConnectionError: Connection error.
Traceback (most recent call last):
File "d:\ragas_demo\ragas_demo1.py", line 145, in
ragas_result = evaluate(
^^^^^^^^^
File "C:\Program Files\Python311\Lib\site-packages\ragas\evaluation.py", line 255, in evaluate
raise e
File "C:\Program Files\Python311\Lib\site-packages\ragas\evaluation.py", line 237, in evaluate
raise ExceptionInRunner()
ragas.exceptions.ExceptionInRunner: The runner thread which was running the jobs raised an
exeception. Read the traceback above to debug it. You can also pass raise_exceptions=False incase you want to show only a warning message instead. Is there any way to find out how to resolve this issue?