Most reproduction_test runs fail with urllib3 _make_request error #79
Description
Hi,
I’m running the following command to execute the reproduction tests:
for num in $(seq 0 39); do echo "Processing ${num}"; python agentless/test/run_reproduction_tests.py \ --run_id="reproduction_test_generation_filter_sample_${num}" \ --test_jsonl="results/swe-bench-lite/reproduction_test_samples/output_${num}_processed_reproduction_test.jsonl" \ --num_workers 1 \ --testing \ --instance_id django__django-10914; done
Most of the runs fail with an error like this:
`
Processing 6
Using run_id: reproduction_test_generation_filter_sample_6
Traceback (most recent call last):
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/connectionpool.py", line 464, in _make_request
self._validate_conn(conn)
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/connectionpool.py", line 1093, in _validate_conn
conn.connect()
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/connection.py", line 790, in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/connection.py", line 969, in _ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/util/ssl_.py", line 480, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/util/ssl_.py", line 524, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/ssl.py", line 517, in wrap_socket
return self.sslsocket_class._create(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/ssl.py", line 1104, in _create
self.do_handshake()
File "/home/user/anaconda3/envs/agentless/lib/python3.11/ssl.py", line 1382, in do_handshake
self._sslobj.do_handshake()
ssl.SSLEOFError: [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1016)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/connectionpool.py", line 787, in urlopen
response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/connectionpool.py", line 488, in _make_request
raise new_e
urllib3.exceptions.SSLError: [SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1016)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/requests/adapters.py", line 667, in send
resp = conn.urlopen(
^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/connectionpool.py", line 841, in urlopen
retries = retries.increment(
^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/urllib3/util/retry.py", line 519, in increment
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='raw.githubusercontent.com', port=443): Max retries exceeded with url: /pylint-dev/pylint/3b2fbaec045697d53bdd4435e59dbfc2b286df4b/requirements_test.txt (Caused by SSLError(SSLEOFError(8, '[SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1016)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/AVR_JAVA_STUDY/agentless/Agentless/agentless/test/run_reproduction_tests.py", line 170, in <module>
main()
File "/home/user/AVR_JAVA_STUDY/agentless/Agentless/agentless/test/run_reproduction_tests.py", line 166, in main
_run_reproduction_tests(args)
File "/home/user/AVR_JAVA_STUDY/agentless/Agentless/agentless/test/run_reproduction_tests.py", line 46, in _run_reproduction_tests
results = run_reproduction_tests(
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/AVR_JAVA_STUDY/agentless/Agentless/agentless/test/run_tests.py", line 386, in run_reproduction_tests
build_env_images(client, instances, force_rebuild, max_workers)
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/swebench/harness/docker_build.py", line 276, in build_env_images
build_base_images(client, dataset, force_rebuild)
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/swebench/harness/docker_build.py", line 175, in build_base_images
test_specs = get_test_specs_from_dataset(dataset)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/swebench/harness/test_spec.py", line 118, in get_test_specs_from_dataset
return list(map(make_test_spec, cast(list[SWEbenchInstance], dataset)))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/swebench/harness/test_spec.py", line 310, in make_test_spec
env_script_list = make_env_script_list(instance, specs, env_name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/swebench/harness/test_spec.py", line 192, in make_env_script_list
reqs = replace_uninstallable_packages_requirements_txt(get_requirements(instance))
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/swebench/harness/utils.py", line 285, in get_requirements
return get_requirements_by_commit(instance["repo"], commit)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/swebench/harness/utils.py", line 226, in get_requirements_by_commit
reqs = requests.get(reqs_url, headers=HEADERS)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/user/anaconda3/envs/agentless/lib/python3.11/site-packages/requests/adapters.py", line 698, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='raw.githubusercontent.com', port=443): Max retries exceeded with url: /pylint-dev/pylint/3b2fbaec045697d53bdd4435e59dbfc2b286df4b/requirements_test.txt (Caused by SSLError(SSLEOFError(8, '[SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol (_ssl.c:1016)')))
`
It seems related to network requests (urllib3._make_request), but I’m not sure if this is expected behavior or an issue with my setup.
Is this error normal when running the reproduction tests?
Or could it be caused by my network?
At the same time, many files results/swe-bench-lite/reproduction_test_samples/output_{i}_processed_reproduction_test_verified.jsonl do not exist, and some that do exist are empty files. Out of 40, only 2 contain content.
Any guidance would be appreciated.
Thanks!