4 simulate.py - simulation script for the 2012 UCC Programming Competition
5 NOTE: This is not the manager program for a stratego game
6 It merely calls the manager program as appropriate, and records results
7 Plays exactly ONE round, but does not overwrite previously played rounds
8 eg: run once to generate round1.results, twice to generate round2.results etc
9 Also generates total.scores based on results from every round.
11 Now (sortof) generates .html files to display results in a prettiful manner.
15 author Sam Moore (matches) [SZM]
16 website http://matches.ucc.asn.au/stratego
18 git git.ucc.asn.au/progcomp2012.git
25 #Global variables/arguments
27 baseDirectory = "../.." #Base directory for results, logs, agents
28 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
33 if len(sys.argv) >= 2:
34 nRounds = int(sys.argv[1])
35 if len(sys.argv) >= 3:
36 nGames = int(sys.argv[2])
38 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
39 if len(sys.argv) >= 4:
40 baseDirectory = sys.argv[3]
41 if len(sys.argv) >= 6:
42 print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
45 resultsDirectory = baseDirectory+"/web/results/" #Where results will go (results are in the form of text files of agent names and scores)
46 logDirectory = baseDirectory+"/web/log/" #Where log files go (direct output of manager program)
47 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
48 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
49 if len(sys.argv) >= 5:
50 managerPath = sys.argv[5]
53 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
54 scores = {"VICTORY":(0.3,0.1, "DEFEAT"), "DEFEAT":(0.1,0.3, "VICTORY"), "SURRENDER":(0,0.3, "VICTORY"), "DRAW":(0.2,0.2, "DRAW"), "DRAW_DEFAULT":(0.1,0.1, "DRAW_DEFAULT"), "ILLEGAL":(-0.1,0.2, "DEFAULT"), "DEFAULT":(0.2,-0.1, "ILLEGAL"), "BOTH_ILLEGAL":(-0.1,-0.1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
57 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
62 #Check the manager program exists TODO: And is executable!
63 if os.path.exists(managerPath) == False:
64 print "Manager program at \""+managerPath+"\" doesn't exist!"
67 #Make necessary directories
69 if os.path.exists(resultsDirectory) == False:
70 os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
73 #Identify the round number by reading from the "info" file in the results directory, if it doesn't exist then start at round 1.
74 if os.path.exists(resultsDirectory+"info") == False:
77 info = open(resultsDirectory+"info", "r")
78 totalRounds = int(info.readline().strip())
80 os.remove(resultsDirectory+"info")
82 info = open(resultsDirectory+"info", "w")
83 info.write(str(totalRounds + nRounds) + "\n")
88 if os.path.exists(logDirectory) == False:
89 os.mkdir(logDirectory) #Make the log directory if it didn't exist
92 startTime = time() #Record time at which simulation starts
96 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
98 print "Simulating one round."
100 print "Identifying possible agents in \""+agentsDirectory+"\""
102 #Get all agent names from agentsDirectory
103 agentNames = os.listdir(agentsDirectory)
105 for name in agentNames:
107 sys.stdout.write("Scan \""+name+"\"... ")
108 if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
110 sys.stdout.write(" Invalid! (Not a directory)\n")
113 if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist
115 sys.stdout.write(" Invalid! (No \"info\" file found)\n")
118 infoFile = open(agentsDirectory+name+"/info", "r")
119 agentExecutable = agentsDirectory+name+"/"+(infoFile.readline().strip())
120 author = infoFile.readline().strip()
121 language = infoFile.readline().strip()
124 line = infoFile.readline()
131 if os.path.exists(agentExecutable.split(" ")[0]) == False:
133 sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
138 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
140 #Convert array of valid names into array of dictionaries containing information about each agent
141 #I'm starting to like python...
142 agents.append({"name":name, "path":agentExecutable, "author":author, "language":language, "description":description, "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
145 print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
148 print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
151 #Prepare the pretty .html files if they don't exist
153 print "Preparing .html results files..."
155 #BACKUP THE RESULTS DIRECTORY GOD DAMMIT
156 os.system("mkdir .before_round"+str(totalRounds)+"_BACKUP/; cp " + resultsDirectory+"*" + " .before_round"+str(totalRounds)+"_BACKUP/")
159 if os.path.exists(resultsDirectory + "index.html") == True:
160 os.remove(resultsDirectory + "index.html") #Delete the file
161 totalFile = open(resultsDirectory + "index.html", "w")
162 totalFile.write("<html>\n<head>\n <title> Round in progress... </title>\n</head>\n<body>\n")
164 totalFile.write("<h1> Rounds " + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + " in progress...</h1>\n")
166 totalFile.write("<h1> Round " + str(totalRounds) + " in progress...</h1>\n")
167 totalFile.write("<p> Please wait for the rounds to finish. You can view the current progress by watching the <a href = \"../log\"/>Log Files</a> </p>")
169 totalFile.write("<h2> Round Summaries </h2>\n")
170 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
171 for i in range(1, totalRounds):
172 totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
173 totalFile.write("</table>\n")
175 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
180 if os.path.exists(resultsDirectory+agent["name"] + ".html") == False:
181 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
182 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " overview</title>\n</head>\n<body>\n<h1> Overview for " + agent["name"]+" </h1>\n")
183 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
184 agentFile.write("<tr> <th> Name </th> <th> Author </th> <th> Language </th> </tr>\n")
185 agentFile.write("<tr> <td> "+agent["name"]+" </td> <td> "+agent["author"]+" </td> <td> "+agent["language"]+" </td> </tr>\n")
186 agentFile.write("</table>\n");
188 agentFile.write("<p> <b>Description</b> </p>\n")
189 agentFile.write("<p> " + agent["description"] + " </p>\n")
192 os.rename(resultsDirectory+agent["name"] + ".html", "tmpfile")
194 oldFile = open("tmpfile", "r")
195 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
196 line = oldFile.readline()
199 # print "Interpreting line \"" + line.strip() + "\""
200 if line.strip() == "</body>" or line.strip() == "<!--end-->":
202 elif line == "<h3> Round Overview </h3>\n":
203 agentFile.write(line)
204 line = oldFile.readline()
205 agentFile.write(line)
206 line = oldFile.readline()
207 if line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
208 #sys.stdout.write("Adding scores... " + line + "\n")
209 agentFile.write(line)
210 line = oldFile.readline()
211 values = line.split(' ')
212 agent["totalScore"] += float(values[2].strip())
213 agent["Wins"] += int(values[5].strip())
214 agent["Losses"] += int(values[8].strip())
215 agent["Draws"] += int(values[11].strip())
216 agent["Illegal"] += int(values[14].strip())
217 agent["Errors"] += int(values[17].strip())
218 agentFile.write(line)
219 line = oldFile.readline()
222 print "Prepared results file \"" + resultsDirectory+agent["name"] + ".html\"."
231 totalGames = nGames/2 * len(agents) * (len(agents)-1)
232 for roundNumber in range(totalRounds, totalRounds + nRounds):
234 if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
235 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
238 agent.update({"name":agent["name"], "path":agent["path"], "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
241 print "Commencing ROUND " + str(roundNumber) + " combat!"
242 print "Total: " + str(totalGames) + " games to be played. This could take a while..."
247 #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
249 for red in agents: #for each agent playing as red,
250 for blue in agents: #against each other agent, playing as blue
252 continue #Exclude battles against self
255 for i in range(1, nGames/2 + 1):
257 gameID = str(roundNumber) + "." + str(gameNumber)
258 #Play a game and read the result. Note the game is logged to a file based on the agent's names
260 sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
261 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(gameID)
262 errorLog = [logDirectory + "error/" + red["name"] + "."+str(gameID), logDirectory + "error/" + blue["name"] + "."+str(gameID)]
263 #Run the game, outputting to logFile; stderr of (both) AI programs is directed to logFile.stderr
264 outline = os.popen(managerPath + " -m 1000 -o " + logFile + " -T " + str(timeoutValue) + " \"" + red["path"] + "\" \"" + blue["path"] + "\" 2>> " + logFile+".stderr", "r").read()
266 #os.system("mv tmp.mp4 " + logFile + ".mp4")
268 #If there were no errors, get rid of the stderr file
269 if os.stat(logFile+".stderr").st_size <= 0:
270 os.remove(logFile+".stderr")
271 results = outline.split(' ')
273 if len(results) != 6:
275 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
276 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
277 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
278 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
279 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
283 if results[1] == "RED":
288 elif results[1] == "BLUE":
295 if results[1] == "BOTH":
296 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
297 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
298 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
299 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
302 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
303 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
304 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
305 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
306 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
307 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
308 #Write scores to raw text files
309 for agent in [endColour, otherColour]:
310 scoreFile = open(resultsDirectory + agent["name"] + ".scores", "a")
311 scoreFile.write(str(agent["totalScore"] + agent["score"][0]) + "\n")
318 sys.stdout.write(" Result \"")
319 for ii in range(1, len(results)):
320 sys.stdout.write(results[ii].strip())
321 if ii < (len(results) - 1):
322 sys.stdout.write(" ")
323 sys.stdout.write("\"\n")
326 print "Completed combat. Total of " + str(gameNumber) + " games played. "
327 if managerErrors != 0:
328 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
332 #We should now have complete score values.
334 print "RESULTS FOR ROUND " + str(roundNumber)
336 #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
337 #for agent in agents:
338 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
340 # print "Agent: " + str(agent)
344 print "Updating pretty .html files... "
347 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
348 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
349 agentFile.write("<h3> Round Overview </h3>\n")
350 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
351 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
352 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
354 agentFile.write("</table>\n")
355 agentFile.write("<p> <a href=round"+str(roundNumber)+".html>Round "+str(roundNumber) + " Scoreboard</a></p>\n")
357 agentFile.write("<h3> Detailed </h3>\n")
358 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
359 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
363 for index in range(0, len(agent["ALL"])):
364 if agent["ALL"][index][4] == "RED":
365 logFile = "../log/round"+str(roundNumber) + "/"+agent["name"]+".vs."+agent["ALL"][index][0]+"."+str(agent["ALL"][index][1])
367 logFile = "../log/round"+str(roundNumber) + "/"+agent["ALL"][index][0]+".vs."+agent["name"]+"."+str(agent["ALL"][index][1])
368 agentFile.write("<tr> <td> <a href="+logFile+">" + str(agent["ALL"][index][1]) + " </a> </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
369 agentFile.write("</table>\n")
371 agent["totalScore"] += agent["score"][0]
372 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
373 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
374 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
375 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
376 agent["Errors"] += len(agent["INTERNAL_ERROR"])
378 agentFile.write("<h3> Accumulated Results </h3>\n")
379 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
380 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
381 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
383 agentFile.write("</table>\n")
392 roundFile = open(resultsDirectory + "round"+str(roundNumber)+".html", "w")
393 roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
394 roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
395 roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
396 roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
397 agents.sort(key = lambda e : e["score"][0], reverse=True)
399 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
400 roundFile.write("</table>\n")
402 command = "cp scores.plt " + resultsDirectory + "scores.plt;"
405 scorePlot = open(resultsDirectory + "scores.plt", "a")
406 scorePlot.write("plot ")
407 for i in range(0, len(agents)):
409 scorePlot.write(", ")
410 scorePlot.write("\""+agents[i]["name"]+".scores\" using ($0+1):1 with linespoints title \""+agents[i]["name"]+"\"")
412 scorePlot.write("\nexit\n")
415 command = "d=$(pwd); cd " + resultsDirectory + ";"
416 command += "gnuplot scores.plt;"
417 command += "rm -f scores.plt;"
418 command += "mv scores.png round"+str(roundNumber)+".png;"
422 roundFile.write("<h2> Accumulated Scores - up to Round " + str(roundNumber)+" </h2>\n")
423 roundFile.write("<img src=\"round"+str(roundNumber)+".png\" alt = \"round"+str(roundNumber)+".png\" title = \"round"+str(roundNumber)+".png\" width = \"640\" height = \"480\"/>\n")
425 roundFile.write("<p> <a href=index.html>Current Scoreboard</a></p>\n")
426 roundFile.write("</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
434 print "Finalising .html files... "
436 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
437 agentFile.write("<!--end-->\n")
439 #Comment out if you don't have gnuplot
441 command = "rm -f " + agent["name"] + ".png;"
442 command += "cp template.plt " + resultsDirectory + agent["name"] + ".plt;"
443 command += "d=$(pwd); cd " + resultsDirectory + ";"
444 command += "sed -i \"s:\[NAME\]:"+agent["name"]+":g\" " +resultsDirectory + agent["name"]+".plt;"
445 command += "gnuplot " + resultsDirectory + agent["name"]+".plt;"
446 command += "rm -f " + resultsDirectory + agent["name"] + ".plt;"
449 agentFile.write("<!--end-->\n")
450 agentFile.write("<h3> Score Graph </h3>\n")
451 agentFile.write("<img src=\""+agent["name"]+".png\" alt=\""+agent["name"]+".png\" title=\""+agent["name"]+".png\" width=\"640\" height=\"480\"/>\n")
454 agentFile.write("<p> <a href=\"index.html\"/>Total Statistics</a> </p>\n")
456 agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
459 if os.path.exists(resultsDirectory + "index.html") == True:
460 os.remove(resultsDirectory + "index.html") #Delete the file
462 totalFile = open(resultsDirectory + "index.html", "w")
463 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
464 totalFile.write("<h1> Total Overview </h1>\n")
465 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
466 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
467 agents.sort(key = lambda e : e["totalScore"], reverse=True)
469 totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
470 totalFile.write("</table>\n")
472 totalFile.write("<h2> Score Graph </h2>\n")
475 command = "d=$(pwd);"
476 command += "cd " + resultsDirectory + ";"
477 command += "rm -f scores.png;"
478 command += "cp round"+str(roundNumber)+".png scores.png;"
482 totalFile.write("<img src=\"scores.png\" alt=\"scores.png\" title=\"scores.png\" width=\"640\" height=\"480\"/>\n")
485 totalFile.write("<h2> Round Summaries </h2>\n")
486 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
487 for i in range(1, totalRounds+1):
488 totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
489 totalFile.write("</table>\n")
491 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
494 #Write results to a raw text file as well
495 textResults = open(resultsDirectory + "total.txt", "w")
497 textResults.write(agent["name"] + " " + str(agent["totalScore"]) + "\n")
504 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."