more robust waitForAvailability

If the DNS change has not propagated yet to all nameservers, then it may
be that the first reachability success goes through but a second one
falis. in that case we are better off first trying for 'consecutive
success' before we proceed with the deploy

can be used in deploy.php like:
```
$testingStage->addBrancherServer("$APP_NAME")
    ->setLabels(['autodestroy=false', 'stage=acceptance', 'ci_ref=' . (\getenv('GITHUB_HEAD_REF') ?: 'none')])
    ->setBrancherReachabilityCheckCount(6)      // Amount of consecutive checks required
    ->setBrancherReachabilityCheckInterval(10); // Amount of seconds between checks
```

Author: tbvdloo

src/Brancher/BrancherHypernodeManager.php

@@ -107,14 +107,20 @@ public function createForHypernode(string $hypernode, array $data = []): string
      *
      * @param string $brancherHypernode Name of the brancher Hypernode
      * @param int $timeout Maximum time to wait for availability
+     * @param int $reachabilityCheckCount Number of consecutive successful checks required
+     * @param int $reachabilityCheckInterval Seconds between reachability checks
      * @return void
      * @throws CreateBrancherHypernodeFailedException
      * @throws HypernodeApiClientException
      * @throws HypernodeApiServerException
      * @throws TimeoutException
      */
-    public function waitForAvailability(string $brancherHypernode, int $timeout = 1500): void
-    {
+    public function waitForAvailability(
+        string $brancherHypernode,
+        int $timeout = 1500,
+        int $reachabilityCheckCount = 6,
+        int $reachabilityCheckInterval = 10
+    ): void {
         $latest = microtime(true);
         $timeElapsed = 0;
         $resolved = false;
@@ -175,7 +181,7 @@ public function waitForAvailability(string $brancherHypernode, int $timeout = 15
             );
         }
 
-        $resolved = false;
+        $consecutiveSuccesses = 0;
         while ($timeElapsed < $timeout) {
             $now = microtime(true);
             $timeElapsed += $now - $latest;
@@ -184,12 +190,37 @@ public function waitForAvailability(string $brancherHypernode, int $timeout = 15
             $connection = @fsockopen(sprintf("%s.hypernode.io", $brancherHypernode), 22);
             if ($connection) {
                 fclose($connection);
-                $resolved = true;
-                break;
+                $consecutiveSuccesses++;
+                $this->log->info(
+                    sprintf(
+                        'Brancher Hypernode %s reachability check %d/%d succeeded.',
+                        $brancherHypernode,
+                        $consecutiveSuccesses,
+                        $reachabilityCheckCount
+                    )
+                );
+
+                if ($consecutiveSuccesses >= $reachabilityCheckCount) {
+                    break;
+                }
+                sleep($reachabilityCheckInterval);
+            } else {
+                if ($consecutiveSuccesses > 0) {
+                    $this->log->info(
+                        sprintf(
+                            'Brancher Hypernode %s reachability check failed, resetting counter (was at %d/%d).',
+                            $brancherHypernode,
+                            $consecutiveSuccesses,
+                            $reachabilityCheckCount
+                        )
+                    );
+                }
+                $consecutiveSuccesses = 0;
+                sleep($reachabilityCheckInterval);
             }
         }
 
-        if (!$resolved) {
+        if ($consecutiveSuccesses < $reachabilityCheckCount) {
             throw new TimeoutException(
                 sprintf('Timed out waiting for brancher Hypernode %s to become reachable', $brancherHypernode)
             );

src/DeployRunner.php

@@ -274,6 +274,8 @@ private function maybeConfigureBrancherServer(Server $server, bool $reuseBranche
         if ($isBrancher && $parentApp) {
             $settings = $serverOptions[Server::OPTION_HN_BRANCHER_SETTINGS] ?? [];
             $labels = $serverOptions[Server::OPTION_HN_BRANCHER_LABELS] ?? [];
+            $reachabilityCheckCount = $serverOptions[Server::OPTION_HN_BRANCHER_REACHABILITY_CHECK_COUNT] ?? 6;
+            $reachabilityCheckInterval = $serverOptions[Server::OPTION_HN_BRANCHER_REACHABILITY_CHECK_INTERVAL] ?? 10;
 
             $this->log->info(sprintf('Creating an brancher Hypernode based on %s.', $parentApp));
             if ($settings) {
@@ -299,7 +301,12 @@ private function maybeConfigureBrancherServer(Server $server, bool $reuseBranche
 
             try {
                 $this->log->info('Waiting for brancher Hypernode to become available...');
-                $this->brancherHypernodeManager->waitForAvailability($brancherApp);
+                $this->brancherHypernodeManager->waitForAvailability(
+                    $brancherApp,
+                    1500,
+                    $reachabilityCheckCount,
+                    $reachabilityCheckInterval
+                );
                 $this->log->info('Brancher Hypernode has become available!');
             } catch (CreateBrancherHypernodeFailedException | TimeoutException $e) {
                 if (in_array($brancherApp, $this->brancherHypernodesRegistered)) {