S3 16MB fix (#4)

* fix(setup): Use latest reedsolo package which can be installed with Python3.10 and Cython

Closes espressif#711

* feat(ci): Publish development releases with custom pipeline

* fix(ci): The development release job should not run by default

Gitlab rules:if adds the job if any of the rules are true.

* fix(ci): Merge two "ci" directories and build_tools into one

* fix(secure download mode): Fix SDM detection on S2/S3

* fix(secure download mode): Reconnect if ROM refuses to respond

Closes espressif#813

* fix(flasher_stub): Correct boundaries for SPIWrite4B and SPIRead4B

SPIWrite4B and SPIRead4B functions are required for flash sizes bigger
than 16MB. This fix corrects the boundaries so SPIWrite and SPIRead
would be used for the whole 16MB area.

The octal flash support for 32 MB chips
(espressif#795) will be added in a
follow-up commit.

Closes espressif#745

Co-authored-by: Roland Dobai <[email protected]>
Co-authored-by: radim.karnis <[email protected]>

.github/workflows/test_esptool.yml

@@ -48,7 +48,7 @@ jobs:
 
         export PATH=$PATH:$ESP8266_BINDIR:$ESP32_BINDIR:$ESP32S2_BINDIR:$ESP32S3_BINDIR:$ESP32C3_BINDIR
 
-        ./test/ci/setup_ci_build_env.sh
+        ./ci/setup_ci_build_env.sh
         make -C flasher_stub V=1
         cd flasher_stub && python ./compare_stubs.py
 

.gitlab-ci.yml

@@ -6,6 +6,7 @@ stages:
   - report
   - build_docs
   - deploy_docs
+  - deploy_development_package
 
 # cache the pip download directory in all jobs
 variables:
@@ -175,7 +176,7 @@ check_stub_build:
 
     TOOLCHAIN_PATHS: "${ESP8266_BINDIR}:${ESP32_BINDIR}:${ESP32S2_BINDIR}:${ESP32S3_BINDIR}:${ESP32C3_BINDIR}"
   script:
-    - ./test/ci/setup_ci_build_env.sh
+    - ./ci/setup_ci_build_env.sh
     - make -C flasher_stub V=1 PATH="${TOOLCHAIN_PATHS}:${PATH}"
     - cd flasher_stub && coverage run --parallel-mode ./compare_stubs.py
 
@@ -366,3 +367,25 @@ deploy_docs_production:
     DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
     DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
     DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esptool"
+
+deploy_dev_package:
+  <<: *test_template
+  rules:
+    - if: $CI_DEV_PUBLISH != null && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  variables:
+    TWINE_NON_INTERACTIVE: "true"
+    TWINE_USERNAME: __token__
+    TWINE_PASSWORD: ${PYPI_ESPTOOL_TOKEN}
+  stage: deploy_development_package
+  dependencies: []
+  script:
+    - python -m pip install --upgrade pip
+    - python -m pip install twine setuptools
+    - python ci/patch_dev_release.py --dev-no ${CI_DEV_PUBLISH} esptool/__init__.py
+    # check what has been changed with git
+    - git diff
+    - python setup.py sdist
+    # skip release if it has already been released (with failure)
+    - python -m pip download esptool==$(python setup.py -V) && exit 1
+    - tar -ztvf dist/*
+    - python -m twine upload dist/*

ci/patch_dev_release.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import argparse
+import re
+
+LINE_RE = re.compile(r"^__version__ = ['\"]([^'\"]*)['\"]")
+NEW_LINE = '__version__ = "{}"'
+
+
+def get_new_version(old_version, dev_number):
+    assert old_version.endswith("-dev")
+    return old_version.replace("-dev", ".dev{}".format(dev_number), 1)
+
+
+def patch_file(path, dev_number):
+    with open(path, "r") as fin:
+        lines = fin.readlines()
+
+    for i, line in enumerate(lines, start=0):
+        m = LINE_RE.search(line)
+        if m:
+            old_version = m.group(1)
+            lines[i] = NEW_LINE.format(get_new_version(old_version, dev_number))
+            break
+
+    with open(path, "w") as fout:
+        fout.writelines(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("file", help="Path to script with __version__")
+    parser.add_argument(
+        "--dev-no", type=int, help="Number N to patch the version to '.devN'"
+    )
+    args = parser.parse_args()
+    patch_file(args.file, args.dev_no)
+
+
+if __name__ == "__main__":
+    main()

esptool/cmds.py

@@ -104,13 +104,13 @@ def detect_chip(
             # cmd not supported on ESP8266 and ESP32 + ESP32-S2 doesn't return chip_id
             if chip_id == cls.IMAGE_CHIP_ID:
                 inst = cls(detect_port._port, baud, trace_enabled=trace_enabled)
-                inst._post_connect()
                 try:
                     inst.read_reg(
                         ESPLoader.CHIP_DETECT_MAGIC_REG_ADDR
                     )  # Dummy read to check Secure Download mode
                 except UnsupportedCommandError:
                     inst.secure_download_mode = True
+                inst._post_connect()
     except (UnsupportedCommandError, struct.error, FatalError) as e:
         # UnsupportedCommmanddError: ESP8266/ESP32 ROM
         # struct.error: ESP32-S2

esptool/loader.py

@@ -672,8 +672,17 @@ def connect(
                         )
             except UnsupportedCommandError:
                 self.secure_download_mode = True
+
+            try:
+                self.check_chip_id()
+            except UnsupportedCommandError:
+                # Fix for ROM not responding in SDM, reconnect and try again
+                if self.secure_download_mode:
+                    self._connect_attempt(mode, usb_jtag_serial, extra_delay)
+                    self.check_chip_id()
+                else:
+                    raise
             self._post_connect()
-            self.check_chip_id()
 
     def _post_connect(self):
         """

esptool/targets/stub_flasher/stub_flasher_32s3beta2.json

@@ -1,4 +1,5 @@
 {
+<<<KP7qNN0CkjjdAcpM3QC+PN0DGjjdAL483QI6PN0BakDdAzZA3QHWQN0BFjTdA8I83QEyQN0CxjzdA8JA3QNqPN0DwkDdAko43QPGON0AvjzdAjo83QKqON0CGjTdAsJE3QCuTN0BijDdAUZM3QGKMN0BijDdAYow3QGKMN0BijDdAYow3QGKMN0BijDdASpE3QGKMN0BFkjdAK5M3QA==",
     "entry": 1077381684,
     "text": "FIADYACAA2BIAMo/BIADYDZBAIH7/wxJwCAAmQjGBAAAgfj/wCAAqAiB9/+goHSICOAIACH2/8AgAIgCJ+jhHfAAAAAIAABgHAAAYBAAAGA2QQAh/P/AIAA4AkH7/8AgACgEICCUnOJB6P9GBAAMODCIAcAgAKgIiASgoHTgCAALImYC6Ib0/yHx/8AgADkCHfAAAOwryz9kq8o/hIAAAEBAAACk68o/8CvLPzZBALH5/yCgdBARIOUqAZYaBoH2/5KhAZCZEZqYwCAAuAmR8/+goHSaiMAgAJIYAJCQ9BvJwMD0wCAAwlgAmpvAIACiSQDAIACSGACB6v+QkPSAgPSHmUeB5f+SoQGQmRGamMAgAMgJoeX/seP/h5wXxgEAfOiHGt7GCADAIACJCsAgALkJRgIAwCAAuQrAIACJCZHX/5qIDAnAIACSWAAd8AAAVCAAYFQwAGA2QQCR/f/AIACICYCAJFZI/5H6/8AgAIgJgIAkVkj/HfAAAAAsIABgACAAYAAAAAg2QQAQESCl/P8h+v8MCMAgAIJiAJH6/4H4/8AgAJJoAMAgAJgIVnn/wCAAiAJ88oAiMCAgBB3wAAAAAEA2QQAQESDl+/8Wav+B7P+R+//AIACSaADAIACYCFZ5/x3wAABYAMo/////AAQgAGCcGgVAaBoFQDZBACH6/0hCFoQJEBEg5ff/FvoIDPgMA0eoDJgigJkQDBiQOIMwMHQQESDl+f8QESCl8v+4IkHu/0CLEYe0LpyT+zsMTLA7swAMQDCwsQwager/4AgAHAjGDgAADEwMGoHn/+AIAAwYxgoAAAAADBqQmgHME4CaAUHf/8AgAIkEgcH/wCAAmQjAIABICFZ0/xwIDBQwhIM4QoAzwDlCOCKKMzkiHfAAAIypBEA2QQAgoiCB/f/gCAAd8AAAcOL6PwggAGBY0gRAeNIEQDZhABARIKXo/zH5/70BrQOB+v/gCABNCgwS7OqIAZKiAJCIEIkBEBEgJe3/kfL/oKIBwCAAiAmgiCDAIACJCbgBrQOB7v/gCACgJIMd8AAA/w8AADZBAIGy/wwZkkgAMJxBmSiR+/85GCk4MDC0miIqMzA8QQwCKVg5SBARICX4/y0KjBoioMUd8AAAtJwEQDZBAIH+/+AIAIIKGAwJgsj8DBKAKZMd8DZBAIH4/+AIAIIKGAwJgsj9DBKAKZMd8DZBABARICX+/yKgAcxqEBEg5fv/LQogIAQd8AD4K8s/9CvLPxAADGBgAAxg//P//zZBABARIKX8/xZaBKH4/4gKvNiB9/+YCLxpsfb/fMzAIACIC5CQFMCIEJCIIMAgAIkLiAqx8f8MOmCqEcAgAJgLoIgQoe7/oJkQkIggwCAAiQsd8FDzzj9EAMo/TADKP1SfBEBAnwRAhKAEQDZhAHzIrQKHky0x9//GBQAAqAMMHL0Bgff/4AgAgfH+ogEAiAjgCACoA4Hz/+AIAOYa3cYKAAAAZgMmDAPNAQwrMmEAge7/4AgAmAGB6P83mQ2oCGYaCDHm/8AgAKJDAJkIHfBAAMo/AADKP+CeBEA2QQAh/P+B3P/IAqgIsfr/gfv/4AgADAiJAh3wUJgEQDZBABARIGXr/4y6gfL/iAiMSBARIKX8/xARIKXo/xYqAKKgBIH2/+AIAB3wIJgEQDZBABARIGXo/7wakeb/iAkbqKkJkeX/DAqKmSJJAILIwQwZgKmDoIB0zIqir0CqIiCYk5zZEBEgZff/RgUArQKB7//gCAAQESCl4v+MShARIOX3/x3wAAA2QQCioMAQESDl+f8d8AAANkEAgqDArQKHkhGioNsQESBl+P+ioNxGBAAAAACCoNuHkggQESAl9/+ioN0QESCl9v8d8DZBADoyxgIAAKICABsiEBEgpfv/N5LxHfAAAADYnwRASEgEQOSfBEBUSARANkEAEBEg5dv/rIoME0GU//AzAYyyqASB9v/gCACtA8YJAK0DgfT/4AgAqASB8//gCAAGCQAQESAl1/8MGPCIASwDoIODrQgWkgCB7P/gCACGAQAAgej/4AgAHfAsGAVA4BUFQDaBAAxLDBqB/P/gCAAcBgYMAAAAYFRDDAgMGtCVEQyNOTHtAolhqVGZQYkhiRHZASwPDMwMS4Hx/+AIAFBEwFozWiLmFM0MAh3wAACgdgNAzOMEQMB2A0AYmQRAQHcDQDZBIWKkHeBmERpmWQYMF1KgAGLREFClIEB3EVJmGhARIKXx/0e3AsZIAK0GgfD/4AgAgqIP0IgRC5MaiJkIxjIAkqIP0JkRGpmYCVBzwCqJkcv+QHdjzQe9Aa0Ch7kJEBEg5fP/hgEAAIHh/+AIAKCgdIyqDAiCZhZ9CEYTAAAAEBEgpeT/vQetARARICXo/xARIKXj/80HELEgYKYggdX/4AgAeiJ6VTe1yIKhB8CIEZKkHRqI4JkRiAgamZgJgHXAl7cCBt7/Bur/AAwIgkZsoqQbEKqggcj/4AgAVgr/sqILogZsELuwEBEgpcAA9+oS9kcPkqINEJmwepmiSQAbd4bx/3zpl5rBZkcSkqEHgiYawJkRGpmJCTe4Aoe1hyKiCxAisL0GrQKBtP/gCAAQESCl2f+tAhwLEBEgJd3/EBEgpdj/DBoQESAl3/8d8DYhIaLREIGl/+AIAIYPAAAMFEBEEVGF/kBDYzqCzQS9Aa0Ch7UKEBEgZeL/xgEAAACBm//gCACgoHT8Os0EvQGi0RCBmP/gCABKIkAzwFbj+yKiCxAisK0CstEQgZT/4AgArQIcCxARIGXV/y0DhgAAIqBjHfAAAHQYBUA2gQAMSwwagXH/4AgALAcGEQAMSwwagfn/4AgAcFRDDAgMFtCVEe0CiUGJMZkhORGJASwPDI0cLAxLrQZpYWlRgWT/4AgADEutBoFg/+AIAFozWiJQRMDmFLcd8OziBEA2YQBBUv5YNFAzYxbTC1gUWlNQXEFGAQAQESBllP9oRKYWBGgkZ6XvEBEgJYz/Fmr/aBRxR/46Vle3bAxLDBqBS//gCABgUHSCoQBQeMB3swjNA70CrQZGDgDNB70CrQZS1f8QESCl8/86VVBYQQwJhgUAAADCoQCZARARICXy/5gBctcBG5mQkHR6pnqyVznjcMPAEBEgpfD/DEsMGoEz/+AIAIYFAADNA70CrQaB0//gCACgoHSMOiKgxClUKBQ6IikUKDQwIsAiZAMd8AAAAADKP09IQUn8K8s/rIA3QJggDGCogjdAJIQ3QAgACGCAIQxgEIA3QBCAA2BQgDdADAAAYDhAAGD//wAALIEAYIyAAAAQQAAAACzLPxAsyz9QAMo/VADKP1wsyz8UAABg8P//APwryz9kK8o/cADKP/hNBEA4SARAKKAEQKyfBEBsOgRAAOEEQHDmBEBIMQRA0LYEQCyjBEAsqQRABFwEQPSLBEDk4QRAeOIEQATiBEBolQRAtPgEQFz6BEDQ+ARALFQDQOxbBEA24QAhzv8MCinBgef/4AgAEBEgJZT/FmoEUVf+MVb+wCAAKAVBU/4pAzFU/sAgAGgDaQR8xmAiEAwmYCIgwCAAKQUoBEFP/kAiEEKkAEAiIMAgACkDxgEASQJLIgYCAAAhtv8xt/8MBDcy7BARIKWp/wxLosEwEBEgJa3/IqEBEBEgZaj/QXf9kCIRKiTAIABJAjGs/yFP/TkCEBEg5Yn/LQoW+gUhTP7BTf6oAgwrgU/+4AgAMaT/saX/HBoMDMAgAKkDgbr/4AgADBrwqgGBqf7gCACxnv+oAgwVgbX/4AgAqAKBof7gCACoAoGy/+AIADGY/8AgACgDUCIgwCAAKQOGGAAQESClgf+8GjGS/xwasZL/wCAAomMAIMIggaP/4AgAMY//DEXAIAAoAwwaUCIgwCAAKQPwqgHGCAAAALGJ/80KDFqBmf/gCAAxhv9SoQHAIAAoAywKUCIgwCAAKQOBhP7gCACBlP/gCAAhf//AIAAoAsy6HMMwIhAiwvgMEyCjgwwLgY3/4AgA8Xj/DB0MHLKgAeKhAEDdEQDMEWC7AaKgAIGG/+AIACFz/ypEIYD9YtIrxhcAAAAAUQj+wCAAMgUAMDB0FtMEDBrwqgHAIAAiRQCBZf7gCACionHAqhGBeP/gCACBd//gCABxYf986MAgADgHfPqAMxAQqgHAIAA5B4Fx/+AIAIFx/+AIAK0CgXD/4AgAwCAAKAQWovkMB8AgADgEDBLAIAB5BCJBJCIDAQwoeaEiQSWCURMcN3cSJBxHdxIhZpIhIgMDcgMCgCIRcCIgZkISKCPAIAAoAimhhgEAAAAcIiJRExARIGWJ/7KgCKLBJBARIOWM/7IDAyIDAoC7ESBbICE7/yAg9FeyGqKgwBARIKWH/6Kg7hARICWH/xARIOWF/wba/yIDARxHJzc39iIbBvoAACLCLyAgdLZCAgYlAHEt/3AioCgCoAIAACLC/iAgdBwnJ7cCRvAAcSf/cCKgKAKgAgBywjBwcHS2V8WG6gAsSQwHIqDAlxUChugAeaEMcq0HEBEgZYD/rQcQESDlf/8QESBlfv8QESAlfv8Mi6LBJCLC/xARIGWB/1Yi/YZFAAwSVuU1wsEQvQWtBYEj/+AIAFbqNBxLosEQEBEgJX//hrEADBJWtTOBHf/gCACgJYPGywAmhQQMEsbJAFgjKDMgdSBwcLRW1/4QESClUP9xCv2s2sb3/1CsQVe3Gb0KDEyioAGBCP3gCADGAwAAItLwUtUQxgIAgQr/4AgAFtr+xuz/zDJGmAAAAFCQ9FZp/HH6/AYNAFCg9Ve3Ir0KwqAEoqABgff84AgAxgUAAHz5AJkRmiIMGQCZEZpVxgIAgfn+4AgAFlr+xtr/DBkAmREnOcJ9BSolxgoAgej8cKxBd7gWvQoMTAwagef84AgARgMActcQRgMAAACB6v7gCAAW6v4GzP8nl9BGdwAMByKgwCaFAsaVAAwHLQcGlAAmtfXGaQAMEia1AsaNALgzoiMCcqAAEBEgZZH/oCeDxogADBlmtV6IQyCpEQwHIqDCh7oCRoYAuFOoI5JhERARIKVE/5IhEaCXgwYOAAwZZrUziEMgqREMByKgwoe6AoZ7ACgzuFOoIyB4gpJhERARIKVB/yG5/AwIkiERiWIi0it5IqCYgy0Jhm4AkbP8DAeiCQAioMZ3mgIGbQB4I7LF8CKgwLeXAShZDAeSoO9GAgB6g4IIGBt3gJkwtyfyggMFcgMEgIgRcIggcgMGAHcRgHcgggMHgIgBcIgggJnAgqDBDAeQKJOGWQCBm/wioMaSCAB9CRaJFZg4DAcioMh3GQLGUgAoWJJIAAZOAByJDAcMEpcVAsZNAPhz6GPYU8hDuDOiIwKBj/7gCAAMCH0KoCiDhkYADBImRQLGQQCoIwwLgYf+4AgABiAAAFCQNAwHIqDAdxkChj0AUFRBi8N8+EYPAKg8gmESkmERwmEQgX7+4AgAwiEQgiESKCx4HKgMkiERcHIQJgINwCAA2AogKDDQIhAgdyDAIAB5ChuZwswQVzm+BpT/ZkUChpL/DAcioMCGJgAMEia1AsYhACFb/ohTeCOJAiFa/nkCDAIGHQChVv4MB+gKDBmyxfCNBy0HsCmT4ImDIIgQIqDGd5hgwVD+fQjYDCKgybc9U7DwFCKgwFavBC0IhgIAACqDiGhLIokHjQkqfiD9wLcy7RZ43fkMeQoGdP8ADBJmhRchQP6IAowYgqDIDAd5AiE8/nkCDBKAJ4MMB0YBAAAMByKg/yCgdBARIGVG/3CgdBARIKVF/xARIGVE/1YStSIDARwnJzce9jICxtD+IsL9ICB0DPcntwJGzf5xK/5wIqAoAqACAHKg0ncSYHKg1HeSAkYhAAbG/gAoMzgjEBEgJSP/jQpWirCionHAqhGCYRKBKv7gCABxHf6RHf7AIAB4B4IhEnC0NcB3EZB3EHC7ICC7gq0IMLvCgSn+4AgAoqPogR7+4AgARrH+AADYU8hDuDOoIxARIKVN/4as/gCyAwMiAwKAuxEguyCyy/CiwxgQESClbv+Gpf4AIgMDcgMCgCIRcCIggRf+4AgAcQf8IsLwiDeAImMWUqeIF4qCgIxBRgMAAACCYRIQESAlAf+CIRKSJwSmGQWSJwKXqOcQESCl+P4Wav+oF80CssMYgQb+4AgAjDoyoMQ5VzgXKjM5Fzg3ICPAKTeBAP7gCACGh/4AAHIDAiLDGDIDAwwZgDMRcDMgMsPwBiMAceL9gZP7mAc5sZCIwIlBiCYMGYezAQw5kmEREBEgZfn+kiERgdr9mQHoB6HZ/d0IILIgwsEs8sEQgmESger94AgAuCadCqixgiESoLvAuSagM8C4B6oiqEEMDKq7DBq5B5DKg4C7wMDQdFaMAMLbgMCtkxZqAa0IgmESkmEREBEgJV3/giESkiERgmcAUcT7eDWMo5CPMZCIwNYoAFb39dapADG/+yKgxylTRgAAjDmcBwZU/hbXlFG6+yKgyClVhlD+ADG3+yKgySlThk3+KCNWEpMQESBlGf+ionHAqhGBtP3gCAAQESBlCP+Bv/3gCAAGRP4oMxaykBARIOUW/6Kj6IGr/eAIABARICUG/+ACAAY8/gAQESBlBf8d8AAANkEAnQKCoMAoA4eZD8wyDBKGBwAMAikDfOKGDwAmEgcmIhiGAwAAAIKg24ApI4eZKgwiKQN88kYIAAAAIqDcJ5kKDBIpAy0IBgQAAACCoN188oeZBgwSKQMioNsd8AAA",
     "text_start": 1077379072,

flasher_stub/stub_commands.c

@@ -119,7 +119,7 @@ void handle_flash_read(uint32_t addr, uint32_t len, uint32_t block_size,
       uint32_t n = len - num_sent;
       if (n > block_size) n = block_size;
       #if defined(ESP32S3)
-        if (addr + len-1 > 0x00ffffff)
+        if (addr + n > 0x01000000)
           res = SPIRead4B(1, SPI_FLASH_FASTRD_MODE, addr, buf, n);
         else
           res = SPIRead(addr, (uint32_t *)buf, n);
@@ -158,7 +158,7 @@ int handle_flash_get_md5sum(uint32_t addr, uint32_t len) {
       n = FLASH_SECTOR_SIZE;
     }
     #if defined(ESP32S3)
-      if (addr + len > 0x00ffffff)
+      if (addr + n > 0x01000000)
         res = SPIRead4B(1, SPI_FLASH_FASTRD_MODE, addr, buf, n);
       else
         res = SPIRead(addr, (uint32_t *)buf, n);

flasher_stub/stub_write_flash.c

@@ -320,7 +320,7 @@ void handle_flash_data(void *data_buf, uint32_t length) {
 
   /* do the actual write */
   #if defined(ESP32S3)
-    if (fs.next_write + length > 0x00ffffff)
+    if (fs.next_write + length > 0x01000000)
       res = SPIWrite4B(1, SPI_FLASH_FASTRD_MODE, fs.next_write, data_buf, length);
     else
       res = SPIWrite(fs.next_write, data_buf, length);

setup.py

@@ -121,7 +121,7 @@ def find_version(*file_paths):
         "cryptography>=2.1.4",
         "ecdsa>=0.16.0",
         "pyserial>=3.0",
-        "reedsolo>=1.5.3,<=1.5.4",
+        "reedsolo>=1.5.3,<=1.6.0",
     ],
     packages=find_packages(),
     include_package_data=True,