Handle additional variations of rate limit error messages (#182)

There were another two variations of a rate limit error message
that previously weren't being detected as an error related to
exceeding the rate limit. This meant Hatchet wouldn't perform
request backoff, which would result in cascading failures.

Fixes #176.

Co-authored-by: Richard Schneeman <[email protected]>

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## HEAD
 
+- Handle additional variations of rate limit error messages (https://github.com/heroku/hatchet/pull/182)
 - Add `HATCHET_DEFAULT_STACK` for configuring the default stack (https://github.com/heroku/hatchet/pull/184)
 - Fix typo in the reaper `"Duplicate destroy attempted"` message (https://github.com/heroku/hatchet/pull/175)
 - Set `init.defaultBranch` in ci:setup to suppress `git init` warning in Git 2.30+ (https://github.com/heroku/hatchet/issues/172)

lib/hatchet/git_app.rb

@@ -13,7 +13,7 @@ def push_without_retry!
           output = git_push_heroku_yall
         rescue FailedDeploy => e
           case e.output
-          when /reached the API rate limit/, /429 Too Many Requests/
+          when /reached the API rate limit/, /429 Too Many Requests/, /HTTP 429/, /HTTP code = 429/
             throw(:throttle)
           else
             raise e unless @allow_failure

spec/unit/shell_throttle_spec.rb

@@ -43,7 +43,7 @@ def throttle.sleep(value)
   end
 
   describe "git push throttle" do
-    it "rate throttles `git push` " do
+    it "rate throttles `git push` with output variation 1" do
       app = Hatchet::GitApp.new("default_ruby")
       def app.git_push_heroku_yall
         @_git_push_heroku_yall_call_count ||= 0
@@ -67,7 +67,7 @@ def app.what_is_git_push_heroku_yall_call_count; @_git_push_heroku_yall_call_cou
       expect(app.what_is_git_push_heroku_yall_call_count).to be(2)
     end
 
-    it "rate throttles `git push` with different output" do
+    it "rate throttles `git push` with output variation 2" do
       app = Hatchet::GitApp.new("default_ruby")
       def app.git_push_heroku_yall
         @_git_push_heroku_yall_call_count ||= 0
@@ -90,5 +90,53 @@ def app.what_is_git_push_heroku_yall_call_count; @_git_push_heroku_yall_call_cou
 
       expect(app.what_is_git_push_heroku_yall_call_count).to be(2)
     end
+
+    it "rate throttles `git push` with output variation 3" do
+      app = Hatchet::GitApp.new("default_ruby")
+      def app.git_push_heroku_yall
+        @_git_push_heroku_yall_call_count ||= 0
+        @_git_push_heroku_yall_call_count += 1
+        if @_git_push_heroku_yall_call_count >= 2
+          "Success"
+        else
+          raise Hatchet::App::FailedDeployError.new(
+            self,
+            "message",
+            output: "error: RPC failed; HTTP 429 curl 22 The requested URL returned error: 429"
+          )
+        end
+      end
+
+      def app.sleep_called?; @sleep_called; end
+      def app.what_is_git_push_heroku_yall_call_count; @_git_push_heroku_yall_call_count; end
+
+      app.push_without_retry!
+
+      expect(app.what_is_git_push_heroku_yall_call_count).to be(2)
+    end
+
+    it "rate throttles `git push` with output variation 4" do
+      app = Hatchet::GitApp.new("default_ruby")
+      def app.git_push_heroku_yall
+        @_git_push_heroku_yall_call_count ||= 0
+        @_git_push_heroku_yall_call_count += 1
+        if @_git_push_heroku_yall_call_count >= 2
+          "Success"
+        else
+          raise Hatchet::App::FailedDeployError.new(
+            self,
+            "message",
+            output: "error: RPC failed; result=22, HTTP code = 429"
+          )
+        end
+      end
+
+      def app.sleep_called?; @sleep_called; end
+      def app.what_is_git_push_heroku_yall_call_count; @_git_push_heroku_yall_call_count; end
+
+      app.push_without_retry!
+
+      expect(app.what_is_git_push_heroku_yall_call_count).to be(2)
+    end
   end
 end