#!/usr/bin/python import os import sys import time from threading import Thread from threading import Timer from optparse import OptionParser import httplib import urlparse import urllib # # redefine these to suit your needs # DEFAULT_THREADS = 1 # # Thread class that takes care of making the requests # class WorkerThread(Thread): def __init__(self, id, data, wait): Thread.__init__(self) self.id = id self.wait = wait self.data = data self.results = [] self.max_request_time = 0 self.min_request_time = 999999 self.avg_time_per_request = 0 self.total_time = 0 self.num_requests = 0 self.num_gets = 0 self.num_posts = 0 self.cont_mode = False # the thread will be checking this variable in order to know # whether it's been (politely) asked to stop. After being requested # to stop, it is supposed to quite processing and nicely present # its data self.force_stop = False def set_cont_mode( self, mode ): self.cont_mode = mode def run(self ): #for line in self.data: i = 0 while (i < len( self.data) and (self.force_stop == False)): # prepare the data line = self.data[i] urldata = urlparse.urlsplit( line['url'] ) request = urldata[2] if urldata[3]: request = request + "?" + urldata[3]; # store the starting time req_start_time = time.time() # make the connection conn = httplib.HTTPConnection( urldata[1] ) if line['type'] == "GET": conn.request( "GET", request ) self.num_gets = self.num_gets + 1 else: headers = { 'Content-Type': 'application/x-www-form-urlencoded' } params = urllib.urlencode( line['form'] ) conn.request( "POST", request, params, headers ) self.num_posts = self.num_posts + 1 # and get the response response = conn.getresponse() # save the number of requests self.num_requests = self.num_requests + 1 # update the max and min request times if needed req_end_time = time.time() req_time_diff = req_end_time - req_start_time if req_time_diff < self.min_request_time: self.min_request_time = req_time_diff if req_time_diff > self.max_request_time: self.max_request_time = req_time_diff # save some data about this request line['id'] = self.num_requests line['req_time'] = req_time_diff line['http_response'] = response.status line['timestamp'] = time.asctime(time.localtime()) # save this request self.results.append( line ) # save the time it took to perform this request self.total_time = self.total_time + req_time_diff # check if we have to wait if self.wait > 0.0: time.sleep( self.wait ) # move to the next line i = i + 1 # unless we're in 'non-stop' mode... if self.cont_mode == True: i = i % len( self.data ) # save the average time self.avg_time_per_request = self.total_time / self.num_requests # # class that loads a script file and parses it # class ScriptParser: def __init__( self, filename ): self.filename = filename self.data = [] self.num_lines = 0 def parse(self): # open the file try: f = open( self.filename, "r" ) except IOError: print "Error opening file!" sys.exit( -1 ) # read each one of its lines, parse it and store it in our # internal structure for line in f: self.data.append( self.parseLine( line )) self.num_lines = self.num_lines + 1 f.close() def parseLine( self, line ): # split the line into its parts parts = line.split( ",", 3 ) result = {} # put everything back together result['type'] = parts[0].strip() result['url'] = parts[1].strip() # check the form vars if result['type'] == "POST": result['form'] = self.parseFormVars( parts[2] ) return result def parseFormVars( self, formVars ): # split the line into its parts parts = formVars.split( "|" ) # and reprocess each one of the form vars result = {} for part in parts: var = part.split( "=", 2 ) result[var[0].strip().replace( "\"", "" )] = var[1].strip() return result # # callback method for the timer, that will tell the threads # that it's time to stop # def timer_callback(): for t in pool: print "Informing thread " + str( t.id ) + " to stop!" t.force_stop = True # # process command line args # parser = OptionParser() parser.add_option( "-n", "--thread-number", type="int", dest="num_threads", default=DEFAULT_THREADS ) #parser.add_option( "-r", "--number-of-runs", type="int", dest="num_runs", default=1 ) parser.add_option( "-w", "--wait", type="int", dest="milliseconds_wait", default=0 ) parser.add_option( "-f", "--file", type="string", dest="filename" ) parser.add_option( "-o", "--outfile", type="string", dest="outfile" ) parser.add_option( "-t", "--time", type="int", dest="run_time", default=0 ) (options, args ) = parser.parse_args() if options.filename == "": print "File name must be provided" sys.exit(-1) loader = ScriptParser( options.filename ) loader.parse() print "=== Parameters ===" print " Number of requests in file: " + str( loader.num_lines ) # # array for the thread pool # pool = [] # show the parameters print " Number of threads: " + str(options.num_threads) print " Milliseconds to wait between request: " + str(options.milliseconds_wait) seconds_wait = options.milliseconds_wait / 1000.0 # check if dumpding thread data to a file logdata = False if options.outfile: print " Logging thread data to file: " + options.outfile logdata = True # mark the time when we start time_start = time.time() # initialize and call threads for t in range( 0, options.num_threads ): thread = WorkerThread( t, loader.data, seconds_wait ) # check if we've given a time frame in minutes to run if options.run_time > 0: thread.set_cont_mode( True ) # append the thread to the pool pool.append( thread ) thread.start() # configure our timer callback to stop the threads after the # given amount of seconds if options.run_time > 0: print "Running for " + str( options.run_time ) + " seconds..." callback = Timer( options.run_time, timer_callback ) callback.start() else: print "Waiting for threads to terminate..." # collect statistics information from the threads total_average_time = 0.0 total_max_time = 0.0 total_min_time = 99999999.0 total_requests = 0 total_gets = 0 total_posts = 0 for t in pool: t.join() # and the time when we end time_end = time.time() # process statistics from all threads if logdata: # log try: print "Writing global data to file " + options.outfile + ".csv" f = open( options.outfile + ".csv", "w" ) f.write( "ThreadId,NumReqs,NumGet,NumPost,TotalTime,AvgTime,MaxTime,MinTime\n" ) except: print "Could not log thread data to file!" logdata = false for t in pool: total_average_time = total_average_time + t.avg_time_per_request total_requests = total_requests + t.num_requests total_gets = total_gets + t.num_gets total_posts = total_posts + t.num_posts if t.min_request_time < total_min_time: total_min_time = t.min_request_time if t.max_request_time > total_max_time: total_max_time = t.max_request_time if logdata: # log global thread data line = str( t.id ) + "," + str( t.num_requests ) + "," + str( t.num_gets) + "," + \ str( t.num_posts ) + "," + str( t.total_time ) + "," + \ str( t.avg_time_per_request ) + "," + str( t.max_request_time ) + "," + \ str( t.min_request_time ) + "\n" f.write( line ) # and thread-specific times filename = options.outfile + "_" + str( t.id ) + ".csv" print "Writing thread " + str( t.id) + " data to file " + filename td = open( filename, "w" ) td.write( "ReqId,Timestamp,Url,HttpResponse,Type,Time\n") for req in t.results: string = str( req['id'] ) + "," + str( req['timestamp'] ) + "," + \ str( req['url'] ) + "," + str( req['http_response'] ) + "," + \ str( req['type'] ) + "," + str( req['req_time'] ) + "\n" td.write( string ) td.close() if logdata: f.close() # calculate the total average total_average_time = total_average_time / options.num_threads # the total time of all threads, but we need to take into account the time # we've spent waiting between request per thread total_time = (time_end - time_start) - (options.num_threads * seconds_wait * total_requests ) # number of requests per second reqs_per_sec = total_requests / total_time # output final information print "total = " + str( total_time ) print "req = " + str( total_requests ) print "gets = " + str( total_gets ) + " (" + str(( float(total_gets) / float(total_requests)) * 100 ) + "%)" print "posts = " + str( total_posts ) + " (" + str(( float(total_posts) / float(total_requests)) * 100 ) + "%)" print "reqs/sec = " + str( reqs_per_sec ) print "avg = " + str( total_average_time ) print "min = " + str( total_min_time ) print "max = " + str( total_max_time )