diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2cf703a..2d7ed5d 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -44,13 +44,12 @@ jobs:
         run: poetry install
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
       - name: Download pandoc
-        run: poetry run python setup_binary.py download_pandoc
+        run: poetry run python binary_build_prep.py
       - name: run tests
         run: poetry run python tests.py
 
   builder_pypandoc:
     needs: [test]
-    if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-20.04  # Any OS is fine as this wheel is not OS dependent
     steps:
       - name: Check out repository
@@ -60,10 +59,12 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.9  # Build any 1 python version as this wheel is not version dependent
-      - name: Update dependencies
-        run: python -m pip install -U pip wheel setuptools
+      - name: Install and configure Poetry
+        run: |
+          pip3 install poetry
+          poetry config virtualenvs.in-project true
       - name: Build wheel
-        run: python setup.py sdist bdist_wheel
+        run: poetry build
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -72,7 +73,6 @@ jobs:
 
   builder_pypandoc_binary:
     needs: [test]
-    if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')
     strategy:
       matrix:
         # Ref: https://cibuildwheel.readthedocs.io/en/stable/options/#archs
@@ -85,12 +85,12 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
-      - name: Remove pyproject and use setuptools
-        run: rm pyproject.toml
       - name: Build binary Archive
-        uses: pypa/cibuildwheel@v2.9.0
+        uses: pypa/cibuildwheel@v2.15.0
         env:
-          CIBW_BEFORE_ALL: "mv setup_binary.py setup.py && python3 setup.py download_pandoc"
+          CIBW_BUILD_FRONTEND: "build"
+          CIBW_ARCHS_MACOS: x86_64 arm64
+          CIBW_BEFORE_ALL: "python3 binary_build_prep.py"
           CIBW_BUILD: cp39-*  # Build any 1 python version as this wheel is not version dependent
           # We skip some variants because:
           # - pandoc does not publish binaries for Linux 32bit
@@ -103,7 +103,7 @@ jobs:
 
   publisher_release:
     needs: [builder_pypandoc, builder_pypandoc_binary]
-    if: startsWith(github.ref, 'refs/tags/v')
+    if: github.ref == 'refs/heads/master' && startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository
diff --git a/binary_build_prep.py b/binary_build_prep.py
new file mode 100644
index 0000000..30da765
--- /dev/null
+++ b/binary_build_prep.py
@@ -0,0 +1,22 @@
+import os
+from pypandoc.pandoc_download import download_pandoc
+
+import subprocess
+import sys
+
+check_result = subprocess.call(["patch", "--dry-run", "-N", "-u", "pyproject.toml", "-i", "pyproject.toml.patch"])
+if check_result != 0:
+    print("Something is wrong with the pyproject.toml patch")
+    sys.exit(1)
+
+
+patch_result = subprocess.check_call(["patch", "-u", "pyproject.toml", "-i", "pyproject.toml.patch"])
+if patch_result != 0:
+    print("Something went wrong when patching pyproject.toml")
+    sys.exit(2)
+
+
+print("Downloading pandoc")
+targetfolder = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pypandoc", "files")
+download_pandoc(targetfolder=targetfolder)
+
diff --git a/build.py b/build.py
new file mode 100644
index 0000000..5801e22
--- /dev/null
+++ b/build.py
@@ -0,0 +1,2 @@
+def build(*args, **kwargs):
+    pass
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index b38a760..c4151d2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,11 +1,11 @@
 [tool.poetry]
 name = "pypandoc"
-version = "1.11"
 description = "Thin wrapper for pandoc"
 authors = ["JessicaTegner <jessica.tegneroutlook.com>"]
 license = "MIT"
 readme = "README.md"
 repository = "https://github.com/JessicaTegner/pypandoc"
+version = "1.12"
 
 classifiers = [
         'Development Status :: 4 - Beta',
@@ -37,8 +37,12 @@ python = "^3.6"
 [tool.poetry.dev-dependencies]
 pandocfilters = "^1.5.0"
 
+[tool.poetry.build]
+script = "build.py"
+generate-setup-file = false
+
 [build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = ["poetry-core>=1.0.0", "setuptools"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.urls]
diff --git a/pyproject.toml.patch b/pyproject.toml.patch
new file mode 100644
index 0000000..d683054
--- /dev/null
+++ b/pyproject.toml.patch
@@ -0,0 +1,10 @@
+--- pyproject.toml	2023-08-31 17:54:06.196415200 +0200
++++ pyproject.binary.toml	2023-08-31 17:57:56.021476600 +0200
+@@ -1,5 +1,6 @@
+ [tool.poetry]
+-name = "pypandoc"
++name = "pypandoc_binary"
++include = ["pypandoc/files/*"]
+ description = "Thin wrapper for pandoc"
+ authors = ["JessicaTegner <jessica.tegneroutlook.com>"]
+ license = "MIT"
diff --git a/setup.py b/setup.py
deleted file mode 100755
index 2765a31..0000000
--- a/setup.py
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-import pypandoc
-import io
-from setuptools import setup
-
-with io.open('README.md', encoding="utf-8") as f:
-    long_description = f.read()
-
-module = pypandoc
-setup(
-    name = 'pypandoc',
-    version = module.__version__,
-    url = pypandoc.__url__,
-    license = pypandoc.__license__,
-    description = pypandoc.__description__,
-    long_description = long_description,
-    long_description_content_type='text/markdown',
-    author = module.__author__.encode('utf8'),
-    author_email = pypandoc.__author_email__,
-    packages = ['pypandoc'],
-    python_requires=pypandoc.__python_requires__,
-    setup_requires = pypandoc.__setup_requires__,
-    classifiers=pypandoc.__classifiers__,
-    test_suite = 'tests'
-)
diff --git a/setup_binary.py b/setup_binary.py
deleted file mode 100644
index bac0945..0000000
--- a/setup_binary.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-import pypandoc
-from setuptools import setup, Command
-
-import sys
-import os
-import os.path
-import io
-
-try:
-    from urllib.request import urlopen
-except ImportError:
-    from urllib import urlopen
-
-
-with io.open('README.md', encoding="utf-8") as f:
-    long_description = f.read()
-
-
-
-class DownloadPandocCommand(Command):
-
-    """Download pandoc"""
-
-    description = "downloads a pandoc release and adds it to the package"
-    user_options = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        from pypandoc.pandoc_download import download_pandoc
-        targetfolder = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pypandoc", "files")
-        download_pandoc(targetfolder=targetfolder, version="2.19.2")
-
-
-cmd_classes = {'download_pandoc': DownloadPandocCommand}
-
-# Make sure wheels end up platform specific, if they include a pandoc binary
-has_pandoc = (os.path.isfile(os.path.join("pypandoc", "files", "pandoc")) or
-              os.path.isfile(os.path.join("pypandoc", "files", "pandoc.exe")))
-is_build_wheel = ("bdist_wheel" in sys.argv)
-is_download_pandoc = ("download_pandoc" in sys.argv)
-
-if is_build_wheel:
-    if has_pandoc or is_download_pandoc:
-        # we need to make sure that bdist_wheel is after is_download_pandoc,
-        # otherwise we don't include pandoc in the wheel... :-(
-        pos_bdist_wheel = sys.argv.index("bdist_wheel")
-        if is_download_pandoc:
-            pos_download_pandoc = sys.argv.index("download_pandoc")
-            if pos_bdist_wheel < pos_download_pandoc:
-                raise RuntimeError("'download_pandoc' needs to be before 'bdist_wheel'.")
-        # we also need to make sure that this version of bdist_wheel supports
-        # the --plat-name argument
-        try:
-            import wheel
-            from distutils.version import StrictVersion
-            if not StrictVersion(wheel.__version__) >= StrictVersion("0.27"):
-                msg = "Including pandoc in wheel needs wheel >=0.27 but found %s.\nPlease update wheel!"
-                raise RuntimeError(msg % wheel.__version__)
-        except ImportError:
-            # the real error will happen further down...
-            print("No wheel installed, please install 'wheel'...")
-        print("forcing platform specific wheel name...")
-        from distutils.util import get_platform
-        sys.argv.insert(pos_bdist_wheel + 1, '--plat-name')
-        sys.argv.insert(pos_bdist_wheel + 2, get_platform())
-    else:
-        print("no pandoc found, building platform unspecific wheel...")
-        print("use 'python setup.py download_pandoc' to download pandoc.")
-
-module = pypandoc
-setup(
-    name = 'pypandoc_binary',
-    version = module.__version__,
-    url = pypandoc.__url__,
-    license = pypandoc.__license__,
-    description = pypandoc.__description__,
-    long_description = long_description,
-    long_description_content_type='text/markdown',
-    author = module.__author__.encode('utf8'),
-    author_email = pypandoc.__author_email__,
-    packages = ['pypandoc'],
-    package_data={'pypandoc': ['files/*']},
-    python_requires=pypandoc.__python_requires__,
-    setup_requires = pypandoc.__setup_requires__,
-    classifiers=pypandoc.__classifiers__,
-    test_suite = 'tests',
-    cmdclass=cmd_classes
-)