From 4d754fa351d0f2f1ac6172d6629a51ec64ddca62 Mon Sep 17 00:00:00 2001 From: Michael Youngstrom Date: Wed, 31 Oct 2018 11:13:12 -0400 Subject: [PATCH] Fix ratelimiting issue and account for other ecs failures --- scripts/xdist/pytest_container_manager.py | 27 ++++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/scripts/xdist/pytest_container_manager.py b/scripts/xdist/pytest_container_manager.py index 4fc22e3397..c7bb7a4f61 100644 --- a/scripts/xdist/pytest_container_manager.py +++ b/scripts/xdist/pytest_container_manager.py @@ -3,6 +3,7 @@ import logging import time import boto3 +from botocore.config import Config from botocore.exceptions import ClientError logging.basicConfig(level=logging.INFO) @@ -13,9 +14,16 @@ class PytestContainerManager(): """ Responsible for spinning up and terminating ECS tasks to be used with pytest-xdist """ + TASK_RUN_TIMEOUT_MINUTES = 10 + MAX_RUN_TASK_RETRIES = 7 def __init__(self, region, cluster): - self.ecs = boto3.client('ecs', region) + config = Config( + retries={ + 'max_attempts': self.MAX_RUN_TASK_RETRIES + } + ) + self.ecs = boto3.client('ecs', region, config=config) self.cluster_name = cluster def spin_up_tasks(self, number_of_tasks, task_name, subnets, security_groups, public_ip, launch_type): @@ -23,9 +31,6 @@ class PytestContainerManager(): Spins up tasks and generates two .txt files, containing the IP/ arns of the new tasks. """ - TASK_RUN_TIMEOUT_MINUTES = 10 - MAX_RUN_TASK_RETRIES = 7 - revision = self.ecs.describe_task_definition(taskDefinition=task_name)['taskDefinition']['revision'] task_definition = "{}:{}".format(task_name, revision) @@ -41,7 +46,7 @@ class PytestContainerManager(): # Spin up tasks. boto3's run_task only allows 10 tasks to be launched at a time task_arns = [] for num in task_num_list: - for retry in range(1, MAX_RUN_TASK_RETRIES + 1): + for retry in range(1, self.MAX_RUN_TASK_RETRIES + 1): try: response = self.ecs.run_task( count=num, @@ -58,9 +63,9 @@ class PytestContainerManager(): ) except ClientError as err: # Handle AWS throttling with an exponential backoff - if retry == MAX_RUN_TASK_RETRIES: + if retry == self.MAX_RUN_TASK_RETRIES: raise StandardError( - "MAX_RUN_TASK_RETRIES ({}) reached while spinning up tasks due to AWS throttling.".format(MAX_RUN_TASK_RETRIES) + "MAX_RUN_TASK_RETRIES ({}) reached while spinning up tasks due to AWS throttling.".format(self.MAX_RUN_TASK_RETRIES) ) logger.info("Hit error: {}. Retrying".format(err)) countdown = 2 ** retry @@ -72,11 +77,17 @@ class PytestContainerManager(): for task_response in response['tasks']: task_arns.append(task_response['taskArn']) + failure_array = response['failures'] + if failure_array: + raise StandardError( + "There was at least one failure when spinning up tasks: {}".format(failure_array) + ) + # Wait for tasks to finish spinning up not_running = task_arns[:] ip_addresses = [] all_running = False - for attempt in range(0, TASK_RUN_TIMEOUT_MINUTES * 2): + for attempt in range(0, self.TASK_RUN_TIMEOUT_MINUTES * 2): time.sleep(30) list_tasks_response = self.ecs.describe_tasks(cluster=self.cluster_name, tasks=not_running)['tasks'] del not_running[:]