4 simulate.py - simulation script for the 2012 UCC Programming Competition
5 NOTE: This is not the manager program for a stratego game
6 It merely calls the manager program as appropriate, and records results
7 Plays exactly ONE round, but does not overwrite previously played rounds
8 eg: run once to generate round1.results, twice to generate round2.results etc
9 Also generates total.scores based on results from every round.
11 Now (sortof) generates .html files to display results in a prettiful manner.
15 author Sam Moore (matches) [SZM]
16 website http://matches.ucc.asn.au/stratego
18 git git.ucc.asn.au/progcomp2012.git
25 #Global variables/arguments
27 baseDirectory = "../.." #Base directory for results, logs, agents
28 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
33 if len(sys.argv) >= 2:
34 nRounds = int(sys.argv[1])
35 if len(sys.argv) >= 3:
36 nGames = int(sys.argv[2])
38 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
39 if len(sys.argv) >= 4:
40 baseDirectory = sys.argv[3]
41 if len(sys.argv) >= 6:
42 print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
45 resultsDirectory = baseDirectory+"/web/results/" #Where results will go (results are in the form of text files of agent names and scores)
46 logDirectory = baseDirectory+"/web/log/" #Where log files go (direct output of manager program)
47 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
48 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
49 if len(sys.argv) >= 5:
50 managerPath = sys.argv[5]
53 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
54 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
57 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
62 #Check the manager program exists TODO: And is executable!
63 if os.path.exists(managerPath) == False:
64 print "Manager program at \""+managerPath+"\" doesn't exist!"
67 #Make necessary directories
69 if os.path.exists(resultsDirectory) == False:
70 os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
73 #Identify the round number by reading from the "info" file in the results directory, if it doesn't exist then start at round 1.
74 if os.path.exists(resultsDirectory+"info") == False:
77 info = open(resultsDirectory+"info", "r")
78 totalRounds = int(info.readline().strip())
80 os.remove(resultsDirectory+"info")
82 info = open(resultsDirectory+"info", "w")
83 info.write(str(totalRounds + nRounds) + "\n")
88 if os.path.exists(logDirectory) == False:
89 os.mkdir(logDirectory) #Make the log directory if it didn't exist
92 startTime = time() #Record time at which simulation starts
96 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
98 print "Simulating one round."
100 print "Identifying possible agents in \""+agentsDirectory+"\""
102 #Get all agent names from agentsDirectory
103 agentNames = os.listdir(agentsDirectory)
105 for name in agentNames:
107 sys.stdout.write("Scan \""+name+"\"... ")
108 if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
110 sys.stdout.write(" Invalid! (Not a directory)\n")
113 if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist
115 sys.stdout.write(" Invalid! (No \"info\" file found)\n")
118 infoFile = open(agentsDirectory+name+"/info", "r")
119 agentExecutable = agentsDirectory+name+"/"+(infoFile.readline().strip())
120 author = infoFile.readline().strip()
121 language = infoFile.readline().strip()
124 line = infoFile.readline()
131 if os.path.exists(agentExecutable.split(" ")[0]) == False:
133 sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
138 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
140 #Convert array of valid names into array of dictionaries containing information about each agent
141 #I'm starting to like python...
142 agents.append({"name":name, "path":agentExecutable, "author":author, "language":language, "description":description, "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
145 print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
148 print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
151 #Prepare the pretty .html files if they don't exist
153 print "Preparing .html results files..."
156 if os.path.exists(resultsDirectory + "index.html") == True:
157 os.remove(resultsDirectory + "index.html") #Delete the file
158 totalFile = open(resultsDirectory + "index.html", "w")
159 totalFile.write("<html>\n<head>\n <title> Round in progress... </title>\n</head>\n<body>\n")
161 totalFile.write("<h1> Rounds " + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + " in progress...</h1>\n")
163 totalFile.write("<h1> Round " + str(totalRounds) + " in progress...</h1>\n")
164 totalFile.write("<p> Please wait for the rounds to finish. You can view the current progress by watching the <a href = \"../log\"/>Log Files</a> </p>")
166 totalFile.write("<h2> Round Summaries </h2>\n")
167 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
168 for i in range(1, totalRounds):
169 totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
170 totalFile.write("</table>\n")
172 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
177 if os.path.exists(resultsDirectory+agent["name"] + ".html") == False:
178 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
179 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " overview</title>\n</head>\n<body>\n<h1> Overview for " + agent["name"]+" </h1>\n")
180 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
181 agentFile.write("<tr> <th> Name </th> <th> Author </th> <th> Language </th> </tr>\n")
182 agentFile.write("<tr> <td> "+agent["name"]+" </td> <td> "+agent["author"]+" </td> <td> "+agent["language"]+" </td> </tr>\n")
183 agentFile.write("</table>\n");
185 agentFile.write("<p> <b>Description</b> </p>\n")
186 agentFile.write("<p> " + agent["description"] + " </p>\n")
189 os.rename(resultsDirectory+agent["name"] + ".html", "tmpfile")
191 oldFile = open("tmpfile", "r")
192 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
193 line = oldFile.readline()
196 # print "Interpreting line \"" + line.strip() + "\""
197 if line.strip() == "</body>" or line.strip() == "<!--end-->":
199 elif line == "<h3> Round Overview </h3>\n":
200 agentFile.write(line)
201 line = oldFile.readline()
202 agentFile.write(line)
203 line = oldFile.readline()
204 if line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
205 #sys.stdout.write("Adding scores... " + line + "\n")
206 agentFile.write(line)
207 line = oldFile.readline()
208 values = line.split(' ')
209 agent["totalScore"] += int(values[2].strip())
210 agent["Wins"] += int(values[5].strip())
211 agent["Losses"] += int(values[8].strip())
212 agent["Draws"] += int(values[11].strip())
213 agent["Illegal"] += int(values[14].strip())
214 agent["Errors"] += int(values[17].strip())
215 agentFile.write(line)
216 line = oldFile.readline()
219 print "Prepared results file \"" + resultsDirectory+agent["name"] + ".html\"."
228 totalGames = nGames/2 * len(agents) * (len(agents)-1)
229 for roundNumber in range(totalRounds, totalRounds + nRounds):
231 if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
232 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
235 agent.update({"name":agent["name"], "path":agent["path"], "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
238 print "Commencing ROUND " + str(roundNumber) + " combat!"
239 print "Total: " + str(totalGames) + " games to be played. This could take a while..."
244 #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
246 for red in agents: #for each agent playing as red,
247 for blue in agents: #against each other agent, playing as blue
249 continue #Exclude battles against self
252 for i in range(1, nGames/2 + 1):
254 gameID = str(roundNumber) + "." + str(gameNumber)
255 #Play a game and read the result. Note the game is logged to a file based on the agent's names
257 sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
258 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(gameID)
259 errorLog = [logDirectory + "error/" + red["name"] + "."+str(gameID), logDirectory + "error/" + blue["name"] + "."+str(gameID)]
260 #Run the game, outputting to logFile; stderr of (both) AI programs is directed to logFile.stderr
261 outline = os.popen(managerPath + " -o " + logFile + " -T " + str(timeoutValue) + " \"" + red["path"] + "\" \"" + blue["path"] + "\" 2>> " + logFile+".stderr", "r").read()
262 #os.system("mv tmp.mp4 " + logFile + ".mp4")
264 #If there were no errors, get rid of the stderr file
265 if os.stat(logFile+".stderr").st_size <= 0:
266 os.remove(logFile+".stderr")
267 results = outline.split(' ')
269 if len(results) != 6:
271 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
272 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
273 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
274 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
275 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
279 if results[1] == "RED":
284 elif results[1] == "BLUE":
291 if results[1] == "BOTH":
292 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
293 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
294 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
295 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
298 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
299 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
300 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
301 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
302 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
303 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
304 #Write scores to raw text files
305 for agent in [endColour, otherColour]:
306 scoreFile = open(resultsDirectory + agent["name"] + ".scores", "a")
307 scoreFile.write(str(agent["totalScore"] + agent["score"][0]) + "\n")
314 sys.stdout.write(" Result \"")
315 for ii in range(1, len(results)):
316 sys.stdout.write(results[ii].strip())
317 if ii < (len(results) - 1):
318 sys.stdout.write(" ")
319 sys.stdout.write("\"\n")
322 print "Completed combat. Total of " + str(gameNumber) + " games played. "
323 if managerErrors != 0:
324 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
328 #We should now have complete score values.
330 print "RESULTS FOR ROUND " + str(roundNumber)
332 #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
333 #for agent in agents:
334 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
336 # print "Agent: " + str(agent)
340 print "Updating pretty .html files... "
343 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
344 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
345 agentFile.write("<h3> Round Overview </h3>\n")
346 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
347 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
348 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
350 agentFile.write("</table>\n")
351 agentFile.write("<p> <a href=round"+str(roundNumber)+".html>Round "+str(roundNumber) + " Scoreboard</a></p>\n")
353 agentFile.write("<h3> Detailed </h3>\n")
354 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
355 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
359 for index in range(0, len(agent["ALL"])):
360 if agent["ALL"][index][4] == "RED":
361 logFile = "../log/round"+str(roundNumber) + "/"+agent["name"]+".vs."+agent["ALL"][index][0]+"."+str(agent["ALL"][index][1])
363 logFile = "../log/round"+str(roundNumber) + "/"+agent["ALL"][index][0]+".vs."+agent["name"]+"."+str(agent["ALL"][index][1])
364 agentFile.write("<tr> <td> <a href="+logFile+">" + str(agent["ALL"][index][1]) + " </a> </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
365 agentFile.write("</table>\n")
367 agent["totalScore"] += agent["score"][0]
368 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
369 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
370 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
371 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
372 agent["Errors"] += len(agent["INTERNAL_ERROR"])
374 agentFile.write("<h3> Accumulated Results </h3>\n")
375 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
376 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
377 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
379 agentFile.write("</table>\n")
388 roundFile = open(resultsDirectory + "round"+str(roundNumber)+".html", "w")
389 roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
390 roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
391 roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
392 roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
393 agents.sort(key = lambda e : e["score"][0], reverse=True)
395 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
396 roundFile.write("</table>\n")
398 command = "cp scores.plt " + resultsDirectory + "scores.plt;"
401 scorePlot = open(resultsDirectory + "scores.plt", "a")
402 scorePlot.write("plot ")
403 for i in range(0, len(agents)):
405 scorePlot.write(", ")
406 scorePlot.write("\""+agents[i]["name"]+".scores\" using ($0+1):1 with linespoints title \""+agents[i]["name"]+"\"")
408 scorePlot.write("\nexit\n")
411 command = "d=$(pwd); cd " + resultsDirectory + ";"
412 command += "gnuplot scores.plt;"
413 command += "rm -f scores.plt;"
414 command += "mv scores.png round"+str(roundNumber)+".png;"
418 roundFile.write("<h2> Accumulated Scores - up to Round " + str(roundNumber)+" </h2>\n")
419 roundFile.write("<img src=\"round"+str(roundNumber)+".png\" alt = \"round"+str(roundNumber)+".png\" title = \"round"+str(roundNumber)+".png\" width = \"640\" height = \"480\"/>\n")
421 roundFile.write("<p> <a href=index.html>Current Scoreboard</a></p>\n")
422 roundFile.write("</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
430 print "Finalising .html files... "
432 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
433 agentFile.write("<!--end-->\n")
435 #Comment out if you don't have gnuplot
437 command = "rm -f " + agent["name"] + ".png;"
438 command += "cp template.plt " + resultsDirectory + agent["name"] + ".plt;"
439 command += "d=$(pwd); cd " + resultsDirectory + ";"
440 command += "sed -i \"s:\[NAME\]:"+agent["name"]+":g\" " +resultsDirectory + agent["name"]+".plt;"
441 command += "gnuplot " + resultsDirectory + agent["name"]+".plt;"
442 command += "rm -f " + resultsDirectory + agent["name"] + ".plt;"
445 agentFile.write("<!--end-->\n")
446 agentFile.write("<h3> Score Graph </h3>\n")
447 agentFile.write("<img src=\""+agent["name"]+".png\" alt=\""+agent["name"]+".png\" title=\""+agent["name"]+".png\" width=\"640\" height=\"480\"/>\n")
450 agentFile.write("<p> <a href=\"index.html\"/>Total Statistics</a> </p>\n")
452 agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
455 if os.path.exists(resultsDirectory + "index.html") == True:
456 os.remove(resultsDirectory + "index.html") #Delete the file
458 totalFile = open(resultsDirectory + "index.html", "w")
459 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
460 totalFile.write("<h1> Total Overview </h1>\n")
461 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
462 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
463 agents.sort(key = lambda e : e["totalScore"], reverse=True)
465 totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
466 totalFile.write("</table>\n")
468 totalFile.write("<h2> Score Graph </h2>\n")
471 command = "d=$(pwd);"
472 command += "cd " + resultsDirectory + ";"
473 command += "rm -f scores.png;"
474 command += "cp round"+str(roundNumber)+".png scores.png;"
478 totalFile.write("<img src=\"scores.png\" alt=\"scores.png\" title=\"scores.png\" width=\"640\" height=\"480\"/>\n")
481 totalFile.write("<h2> Round Summaries </h2>\n")
482 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
483 for i in range(1, totalRounds+1):
484 totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
485 totalFile.write("</table>\n")
487 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
490 #Write results to a raw text file as well
491 textResults = open(resultsDirectory + "total.txt", "w")
493 textResults.write(agent["name"] + " " + str(agent["totalScore"]) + "\n")
500 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."