python: boto3 : multi-threading
This script takes a list of AWS account numbers and scans each one for deployed AWS services, then writes the data to a table. The icstools.py is the library outlined here. This is my first multi-threaded script, which cut down the task of scanning 275 AWS accounts from 4 hours to 12 minutes.
!/usr/local/bin/python3
""" flow control for AWS account service interrogation, using multithreading to speed up the process. """
from datetime import datetime
from multiprocessing import Process
import time
import boto3
import icstools
tempDateTime = datetime.now().strftime("%Y-%m-%d")
dynamodb = boto3.client('dynamodb')
tablename = 'awsAccountAudit' + '-' + str(tempDateTime)
table = icstools.createTable(dynamodb=dynamodb, tablename=tablename)
time.sleep(30) # give time for table to instantiate
open removed-for-privacy/has-readOnly-role.txt as list
s3 = boto3.resource('s3')
obj = s3.Object('removed-for-privacy', 'has-readOnly-role.txt')
body = obj.get()['Body'].read()
hasRoleList = body.decode().split('\n')
def accountIterate(session, id):
question = icstools.AwsAccount(session, id)
try:
result = question.ec2Services()
icstools.writeDB(dynamodb, tablename, result)
except:
print('no ec2 results in ', id)
--- truncated for brevity. each service in library is called and output written to table
# multiprocess array
procs = []
for newID in hasRoleList:
time.sleep(1)
session = icstools.assumeRoleSession(newID)
# create processes for each account iteration
proc = Process(target=accountIterate, args=(session, newID))
procs.append(proc)
proc.start()
run each proc in list of procs
for proc in procs:
proc.join()