From 6ced81761612fe33e339a09362042cb65ac388cb Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 20 Jul 2025 11:33:23 -0400 Subject: [PATCH 01/10] Add no-GIL interpreter support Add `pytest-run-parallel` as dependency, test no-GIL interpreters in CI, and mark Cython module as safe for freethreaded interpreters. --- .github/workflows/test.yml | 6 ++--- .github/workflows/wheel.yml | 3 ++- msgpack/_cmsgpack.pyx | 2 +- msgpack/_packer.pyx | 2 +- msgpack/_unpacker.pyx | 2 +- requirements.txt | 1 + test/test_multithreading.py | 50 +++++++++++++++++++++++++++++++++++++ 7 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 test/test_multithreading.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23d221c8..b84611dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14", "3.14t", "3.13", "3.13t", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -41,12 +41,12 @@ jobs: - name: Test (C extension) shell: bash run: | - pytest -v test + PYTHON_GIL=0 pytest -v test - name: Test (pure Python fallback) shell: bash run: | - MSGPACK_PUREPYTHON=1 pytest -v test + PYTHON_GIL=0 MSGPACK_PUREPYTHON=1 pytest -v test - name: build packages shell: bash diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 686d7dd0..aa60b3cb 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -32,8 +32,9 @@ jobs: uses: pypa/cibuildwheel@v2.23.3 env: CIBW_TEST_REQUIRES: "pytest" - CIBW_TEST_COMMAND: "pytest {package}/test" + CIBW_TEST_COMMAND: "PYTHON_GIL=0 pytest {package}/test" CIBW_SKIP: "pp* cp38-macosx_*" + CIBW_ENABLE: cpython-freerelease - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx index 1faaac3a..41340ee0 100644 --- a/msgpack/_cmsgpack.pyx +++ b/msgpack/_cmsgpack.pyx @@ -1,5 +1,5 @@ # coding: utf-8 -#cython: embedsignature=True, c_string_encoding=ascii, language_level=3 +#cython: embedsignature=True, c_string_encoding=ascii, language_level=3, freethreading_compatible=True from cpython.datetime cimport import_datetime, datetime_new import_datetime() diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 402b6946..94d6600b 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -1,5 +1,5 @@ # coding: utf-8 - +# cython: freethreading_compatible = True from cpython cimport * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact from cpython.datetime cimport ( diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 34ff3304..717666d6 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -1,5 +1,5 @@ # coding: utf-8 - +# cython: freethreading_compatible = True from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject diff --git a/requirements.txt b/requirements.txt index 78a2f38f..5ad4c0f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ Cython~=3.1.1 +pytest-run-parallel[psutil] diff --git a/test/test_multithreading.py b/test/test_multithreading.py new file mode 100644 index 00000000..911420e1 --- /dev/null +++ b/test/test_multithreading.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +from msgpack import Packer +import threading + + +def run_threaded( + func, + num_threads=8, + pass_count=False, + pass_barrier=False, + outer_iterations=1, + prepare_args=None, +): + """Runs a function many times in parallel""" + for _ in range(outer_iterations): + with ThreadPoolExecutor(max_workers=num_threads) as tpe: + if prepare_args is None: + args = [] + else: + args = prepare_args() + if pass_barrier: + barrier = threading.Barrier(num_threads) + args.append(barrier) + if pass_count: + all_args = [(func, i, *args) for i in range(num_threads)] + else: + all_args = [(func, *args) for i in range(num_threads)] + try: + futures = [] + for arg in all_args: + futures.append(tpe.submit(*arg)) + finally: + if len(futures) < num_threads and pass_barrier: + barrier.abort() + for f in futures: + f.result() + + +def test_multithread_packing(): + output = [] + test_data = "abcd" * 10_000_000 + packer = Packer() + + def closure(b): + data = packer.pack(test_data) + output.append(data) + b.wait() + + run_threaded(closure, num_threads=10, pass_barrier=True, pass_count=False) From 75cdd03f28ec72c049ff41657e5871cac6ac8ac5 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 20 Jul 2025 11:40:14 -0400 Subject: [PATCH 02/10] Exclude PYTHON_GIL=0 for now --- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b84611dd..37dfb2af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,7 +41,7 @@ jobs: - name: Test (C extension) shell: bash run: | - PYTHON_GIL=0 pytest -v test + pytest -v test - name: Test (pure Python fallback) shell: bash diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index aa60b3cb..febd05e9 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -32,7 +32,7 @@ jobs: uses: pypa/cibuildwheel@v2.23.3 env: CIBW_TEST_REQUIRES: "pytest" - CIBW_TEST_COMMAND: "PYTHON_GIL=0 pytest {package}/test" + CIBW_TEST_COMMAND: "pytest {package}/test" CIBW_SKIP: "pp* cp38-macosx_*" CIBW_ENABLE: cpython-freerelease From cafe4f0985f3712882d26ca45bddf8c49f6adca7 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 20 Jul 2025 11:41:57 -0400 Subject: [PATCH 03/10] Lint --- test/test_buffer.py | 4 ++-- test/test_multithreading.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test_buffer.py b/test/test_buffer.py index 2c5a14c5..ca097222 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -17,7 +17,7 @@ def test_unpack_bytearray(): obj = unpackb(buf, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_unpack_memoryview(): @@ -26,7 +26,7 @@ def test_unpack_memoryview(): obj = unpackb(view, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_packer_getbuffer(): diff --git a/test/test_multithreading.py b/test/test_multithreading.py index 911420e1..6694fdc6 100644 --- a/test/test_multithreading.py +++ b/test/test_multithreading.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 +import threading from concurrent.futures import ThreadPoolExecutor + from msgpack import Packer -import threading def run_threaded( From 3cc2a384d5c525be7845de9391af0896be2de9bf Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Fri, 25 Jul 2025 14:50:49 +0000 Subject: [PATCH 04/10] Exclude PYTHON_GIL for now --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 37dfb2af..e9c25a26 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ jobs: - name: Test (pure Python fallback) shell: bash run: | - PYTHON_GIL=0 MSGPACK_PUREPYTHON=1 pytest -v test + MSGPACK_PUREPYTHON=1 pytest -v test - name: build packages shell: bash From 61aa23fa0b6d1a496bdb3c9094932f77852a88de Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 6 Aug 2025 21:44:32 +0000 Subject: [PATCH 05/10] Update .github/workflows/wheel.yml Co-authored-by: Lysandros Nikolaou --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index febd05e9..c8b8b520 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -34,7 +34,7 @@ jobs: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" CIBW_SKIP: "pp* cp38-macosx_*" - CIBW_ENABLE: cpython-freerelease + CIBW_ENABLE: cpython-freethreading - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' From d965025f2e0dee80908f0f09ad8e2fba97730e53 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 6 Aug 2025 21:50:26 +0000 Subject: [PATCH 06/10] Fix extremely silly typo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 50babbf2..989cb42a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.14=dev", "3.14t-dev", "3.13", "3.13t", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.14t-dev", "3.13", "3.13t", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} From bf580572ff1cedf8275a5e7c42bafca8d77e6f54 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 7 Aug 2025 14:51:17 +0000 Subject: [PATCH 07/10] Drop 3.8 support in test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 989cb42a..2ca1a6e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.14-dev", "3.14t-dev", "3.13", "3.13t", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.14t-dev", "3.13", "3.13t", "3.12", "3.11", "3.10", "3.9"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} From 42f4cba85e52825e8737a30e14416a0f46ea8d48 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 7 Aug 2025 14:54:02 +0000 Subject: [PATCH 08/10] Drop pytest-run-parallel as test requirement in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c2c93bf7..301c8767 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ Cython~=3.1.2 -pytest-run-parallel[psutil] +#pytest-run-parallel[psutil] From d0797bd27234a5a1e6fb62c49d10a349c1a9ea09 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 7 Aug 2025 15:00:50 +0000 Subject: [PATCH 09/10] Separate C extension and fallback tests for GIL and no-GIL interpreters --- .github/workflows/test.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ca1a6e9..f080daeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,11 @@ jobs: pip install -U pip pip install -r requirements.txt pytest + - name: Install pytest-run-parallel under free-threading + if: contains(matrix.py, 't') + run: | + pip install pytest-run-parallel + - name: Build shell: bash run: | @@ -39,15 +44,29 @@ jobs: pip install . - name: Test (C extension) + if: ! contains(matrix.py, 't') shell: bash run: | pytest -v test - name: Test (pure Python fallback) + if: ! contains(matrix.py, 't') shell: bash run: | MSGPACK_PUREPYTHON=1 pytest -v test + - name: Test (C extension) in parallel under free-threading + if: contains(matrix.py, 't') + shell: bash + run: | + pytest -v --parallel-threads=auto --iterations=20 test + + - name: Test (pure Python fallback) in parallel under free-threading + if: contains(matrix.py, 't') + shell: bash + run: | + MSGPACK_PUREPYTHON=1 pytest -v --parallel-threads=auto --iterations=20 test + - name: build packages shell: bash run: | From 79e5de103a07b7e23f5d6c77f6c94300634bb28e Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 7 Aug 2025 15:04:31 +0000 Subject: [PATCH 10/10] Work around Windows wonkiness On Windows, attempting to upgrade `pip` within CI always yields this: ``` Run pip install -U pip Requirement already satisfied: pip in c:\hostedtoolcache\windows\python\3.10.11\x64\lib\site-packages (25.1.1) Collecting pip Downloading pip-25.2-py3-none-any.whl.metadata (4.7 kB) Downloading pip-25.2-py3-none-any.whl (1.8 MB) ---------------------------------------- 1.8/1.8 MB 48.1 MB/s eta 0:00:00 Notice: A new release of pip is available: 25.1.1 -> 25.2 Notice: To update, run: python.exe -m pip install --upgrade pip ERROR: To modify pip, please run the following command: C:\hostedtoolcache\windows\Python\3.10.11\x64\python.exe -m pip install -U pip Error: Process completed with exit code 1. ``` --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f080daeb..0bcbaebf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,8 @@ jobs: - name: Prepare shell: bash run: | - pip install -U pip + # TODO: Workaround for Windows tests failing when upgrading `pip` with exit code 1 + # pip install -U pip pip install -r requirements.txt pytest - name: Install pytest-run-parallel under free-threading