4 simulate.py - simulation script for the 2012 UCC Programming Competition
5 NOTE: This is not the manager program for a stratego game
6 It merely calls the manager program as appropriate, and records results
7 Plays exactly ONE round, but does not overwrite previously played rounds
8 eg: run once to generate round1.results, twice to generate round2.results etc
9 Also generates total.scores based on results from every round.
11 Now (sortof) generates .html files to display results in a prettiful manner.
15 author Sam Moore (matches) [SZM]
16 website http://matches.ucc.asn.au/stratego
18 git git.ucc.asn.au/progcomp2012.git
25 #Global variables/arguments
27 baseDirectory = "../.." #Base directory for results, logs, agents
28 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
33 if len(sys.argv) >= 2:
34 nRounds = int(sys.argv[1])
35 if len(sys.argv) >= 3:
36 nGames = int(sys.argv[2])
38 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
39 if len(sys.argv) >= 4:
40 baseDirectory = sys.argv[3]
41 if len(sys.argv) >= 6:
42 print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
45 resultsDirectory = baseDirectory+"/web/results/" #Where results will go (results are in the form of text files of agent names and scores)
46 logDirectory = baseDirectory+"/web/log/" #Where log files go (direct output of manager program)
47 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
48 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
49 if len(sys.argv) >= 5:
50 managerPath = sys.argv[5]
53 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
54 scores = {"VICTORY":(0.3,0.1, "DEFEAT"), "DEFEAT":(0.1,0.3, "VICTORY"), "SURRENDER":(0,0.3, "VICTORY"), "DRAW":(0.2,0.2, "DRAW"), "DRAW_DEFAULT":(0.1,0.1, "DRAW_DEFAULT"), "ILLEGAL":(-0.1,0.2, "DEFAULT"), "DEFAULT":(0.2,-0.1, "ILLEGAL"), "BOTH_ILLEGAL":(-0.1,-0.1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
57 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
62 #Check the manager program exists TODO: And is executable!
63 if os.path.exists(managerPath) == False:
64 print "Manager program at \""+managerPath+"\" doesn't exist!"
67 #Make necessary directories
69 if os.path.exists(resultsDirectory) == False:
70 os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
73 #Identify the round number by reading from the "info" file in the results directory, if it doesn't exist then start at round 1.
74 if os.path.exists(resultsDirectory+"info") == False:
77 info = open(resultsDirectory+"info", "r")
78 totalRounds = int(info.readline().strip())
80 os.remove(resultsDirectory+"info")
82 info = open(resultsDirectory+"info", "w")
83 info.write(str(totalRounds + nRounds) + "\n")
88 if os.path.exists(logDirectory) == False:
89 os.mkdir(logDirectory) #Make the log directory if it didn't exist
92 startTime = time() #Record time at which simulation starts
96 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
98 print "Simulating one round."
100 print "Identifying possible agents in \""+agentsDirectory+"\""
102 #Get all agent names from agentsDirectory
103 agentNames = os.listdir(agentsDirectory)
105 for name in agentNames:
107 sys.stdout.write("Scan \""+name+"\"... ")
108 if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
110 sys.stdout.write(" Invalid! (Not a directory)\n")
113 if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist
115 sys.stdout.write(" Invalid! (No \"info\" file found)\n")
118 infoFile = open(agentsDirectory+name+"/info", "r")
119 agentExecutable = agentsDirectory+name+"/"+(infoFile.readline().strip())
120 author = infoFile.readline().strip()
121 language = infoFile.readline().strip()
124 line = infoFile.readline()
131 if os.path.exists(agentExecutable.split(" ")[0]) == False:
133 sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
138 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
140 #Convert array of valid names into array of dictionaries containing information about each agent
141 #I'm starting to like python...
142 agents.append({"name":name, "path":agentExecutable, "author":author, "language":language, "description":description, "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
145 print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
148 print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
151 #Prepare the pretty .html files if they don't exist
153 print "Preparing .html results files..."
156 if os.path.exists(resultsDirectory + "index.html") == True:
157 os.remove(resultsDirectory + "index.html") #Delete the file
158 totalFile = open(resultsDirectory + "index.html", "w")
159 totalFile.write("<html>\n<head>\n <title> Round in progress... </title>\n</head>\n<body>\n")
161 totalFile.write("<h1> Rounds " + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + " in progress...</h1>\n")
163 totalFile.write("<h1> Round " + str(totalRounds) + " in progress...</h1>\n")
164 totalFile.write("<p> Please wait for the rounds to finish. You can view the current progress by watching the <a href = \"../log\"/>Log Files</a> </p>")
166 totalFile.write("<h2> Round Summaries </h2>\n")
167 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
168 for i in range(1, totalRounds):
169 totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
170 totalFile.write("</table>\n")
172 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
177 if os.path.exists(resultsDirectory+agent["name"] + ".html") == False:
178 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
179 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " overview</title>\n</head>\n<body>\n<h1> Overview for " + agent["name"]+" </h1>\n")
180 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
181 agentFile.write("<tr> <th> Name </th> <th> Author </th> <th> Language </th> </tr>\n")
182 agentFile.write("<tr> <td> "+agent["name"]+" </td> <td> "+agent["author"]+" </td> <td> "+agent["language"]+" </td> </tr>\n")
183 agentFile.write("</table>\n");
185 agentFile.write("<p> <b>Description</b> </p>\n")
186 agentFile.write("<p> " + agent["description"] + " </p>\n")
189 os.rename(resultsDirectory+agent["name"] + ".html", "tmpfile")
191 oldFile = open("tmpfile", "r")
192 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
193 line = oldFile.readline()
196 # print "Interpreting line \"" + line.strip() + "\""
197 if line.strip() == "</body>" or line.strip() == "<!--end-->":
199 elif line == "<h3> Round Overview </h3>\n":
200 agentFile.write(line)
201 line = oldFile.readline()
202 agentFile.write(line)
203 line = oldFile.readline()
204 if line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
205 #sys.stdout.write("Adding scores... " + line + "\n")
206 agentFile.write(line)
207 line = oldFile.readline()
208 values = line.split(' ')
209 agent["totalScore"] += float(values[2].strip())
210 agent["Wins"] += int(values[5].strip())
211 agent["Losses"] += int(values[8].strip())
212 agent["Draws"] += int(values[11].strip())
213 agent["Illegal"] += int(values[14].strip())
214 agent["Errors"] += int(values[17].strip())
215 agentFile.write(line)
216 line = oldFile.readline()
219 print "Prepared results file \"" + resultsDirectory+agent["name"] + ".html\"."
228 totalGames = nGames/2 * len(agents) * (len(agents)-1)
229 for roundNumber in range(totalRounds, totalRounds + nRounds):
231 if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
232 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
235 agent.update({"name":agent["name"], "path":agent["path"], "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
238 print "Commencing ROUND " + str(roundNumber) + " combat!"
239 print "Total: " + str(totalGames) + " games to be played. This could take a while..."
244 #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
246 for red in agents: #for each agent playing as red,
247 for blue in agents: #against each other agent, playing as blue
249 continue #Exclude battles against self
252 for i in range(1, nGames/2 + 1):
254 gameID = str(roundNumber) + "." + str(gameNumber)
255 #Play a game and read the result. Note the game is logged to a file based on the agent's names
257 sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
258 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(gameID)
259 errorLog = [logDirectory + "error/" + red["name"] + "."+str(gameID), logDirectory + "error/" + blue["name"] + "."+str(gameID)]
260 #Run the game, outputting to logFile; stderr of (both) AI programs is directed to logFile.stderr
261 outline = os.popen(managerPath + " -m 1000 -o " + logFile + " -T " + str(timeoutValue) + " \"" + red["path"] + "\" \"" + blue["path"] + "\" 2>> " + logFile+".stderr", "r").read()
263 #os.system("mv tmp.mp4 " + logFile + ".mp4")
265 #If there were no errors, get rid of the stderr file
266 if os.stat(logFile+".stderr").st_size <= 0:
267 os.remove(logFile+".stderr")
268 results = outline.split(' ')
270 if len(results) != 6:
272 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
273 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
274 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
275 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
276 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
280 if results[1] == "RED":
285 elif results[1] == "BLUE":
292 if results[1] == "BOTH":
293 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
294 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
295 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
296 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
299 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
300 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
301 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
302 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
303 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
304 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
305 #Write scores to raw text files
306 for agent in [endColour, otherColour]:
307 scoreFile = open(resultsDirectory + agent["name"] + ".scores", "a")
308 scoreFile.write(str(agent["totalScore"] + agent["score"][0]) + "\n")
315 sys.stdout.write(" Result \"")
316 for ii in range(1, len(results)):
317 sys.stdout.write(results[ii].strip())
318 if ii < (len(results) - 1):
319 sys.stdout.write(" ")
320 sys.stdout.write("\"\n")
323 print "Completed combat. Total of " + str(gameNumber) + " games played. "
324 if managerErrors != 0:
325 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
329 #We should now have complete score values.
331 print "RESULTS FOR ROUND " + str(roundNumber)
333 #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
334 #for agent in agents:
335 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
337 # print "Agent: " + str(agent)
341 print "Updating pretty .html files... "
344 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
345 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
346 agentFile.write("<h3> Round Overview </h3>\n")
347 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
348 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
349 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
351 agentFile.write("</table>\n")
352 agentFile.write("<p> <a href=round"+str(roundNumber)+".html>Round "+str(roundNumber) + " Scoreboard</a></p>\n")
354 agentFile.write("<h3> Detailed </h3>\n")
355 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
356 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
360 for index in range(0, len(agent["ALL"])):
361 if agent["ALL"][index][4] == "RED":
362 logFile = "../log/round"+str(roundNumber) + "/"+agent["name"]+".vs."+agent["ALL"][index][0]+"."+str(agent["ALL"][index][1])
364 logFile = "../log/round"+str(roundNumber) + "/"+agent["ALL"][index][0]+".vs."+agent["name"]+"."+str(agent["ALL"][index][1])
365 agentFile.write("<tr> <td> <a href="+logFile+">" + str(agent["ALL"][index][1]) + " </a> </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
366 agentFile.write("</table>\n")
368 agent["totalScore"] += agent["score"][0]
369 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
370 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
371 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
372 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
373 agent["Errors"] += len(agent["INTERNAL_ERROR"])
375 agentFile.write("<h3> Accumulated Results </h3>\n")
376 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
377 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
378 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
380 agentFile.write("</table>\n")
389 roundFile = open(resultsDirectory + "round"+str(roundNumber)+".html", "w")
390 roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
391 roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
392 roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
393 roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
394 agents.sort(key = lambda e : e["score"][0], reverse=True)
396 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
397 roundFile.write("</table>\n")
399 command = "cp scores.plt " + resultsDirectory + "scores.plt;"
402 scorePlot = open(resultsDirectory + "scores.plt", "a")
403 scorePlot.write("plot ")
404 for i in range(0, len(agents)):
406 scorePlot.write(", ")
407 scorePlot.write("\""+agents[i]["name"]+".scores\" using ($0+1):1 with linespoints title \""+agents[i]["name"]+"\"")
409 scorePlot.write("\nexit\n")
412 command = "d=$(pwd); cd " + resultsDirectory + ";"
413 command += "gnuplot scores.plt;"
414 command += "rm -f scores.plt;"
415 command += "mv scores.png round"+str(roundNumber)+".png;"
419 roundFile.write("<h2> Accumulated Scores - up to Round " + str(roundNumber)+" </h2>\n")
420 roundFile.write("<img src=\"round"+str(roundNumber)+".png\" alt = \"round"+str(roundNumber)+".png\" title = \"round"+str(roundNumber)+".png\" width = \"640\" height = \"480\"/>\n")
422 roundFile.write("<p> <a href=index.html>Current Scoreboard</a></p>\n")
423 roundFile.write("</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
431 print "Finalising .html files... "
433 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
434 agentFile.write("<!--end-->\n")
436 #Comment out if you don't have gnuplot
438 command = "rm -f " + agent["name"] + ".png;"
439 command += "cp template.plt " + resultsDirectory + agent["name"] + ".plt;"
440 command += "d=$(pwd); cd " + resultsDirectory + ";"
441 command += "sed -i \"s:\[NAME\]:"+agent["name"]+":g\" " +resultsDirectory + agent["name"]+".plt;"
442 command += "gnuplot " + resultsDirectory + agent["name"]+".plt;"
443 command += "rm -f " + resultsDirectory + agent["name"] + ".plt;"
446 agentFile.write("<!--end-->\n")
447 agentFile.write("<h3> Score Graph </h3>\n")
448 agentFile.write("<img src=\""+agent["name"]+".png\" alt=\""+agent["name"]+".png\" title=\""+agent["name"]+".png\" width=\"640\" height=\"480\"/>\n")
451 agentFile.write("<p> <a href=\"index.html\"/>Total Statistics</a> </p>\n")
453 agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
456 if os.path.exists(resultsDirectory + "index.html") == True:
457 os.remove(resultsDirectory + "index.html") #Delete the file
459 totalFile = open(resultsDirectory + "index.html", "w")
460 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
461 totalFile.write("<h1> Total Overview </h1>\n")
462 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
463 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
464 agents.sort(key = lambda e : e["totalScore"], reverse=True)
466 totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
467 totalFile.write("</table>\n")
469 totalFile.write("<h2> Score Graph </h2>\n")
472 command = "d=$(pwd);"
473 command += "cd " + resultsDirectory + ";"
474 command += "rm -f scores.png;"
475 command += "cp round"+str(roundNumber)+".png scores.png;"
479 totalFile.write("<img src=\"scores.png\" alt=\"scores.png\" title=\"scores.png\" width=\"640\" height=\"480\"/>\n")
482 totalFile.write("<h2> Round Summaries </h2>\n")
483 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
484 for i in range(1, totalRounds+1):
485 totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
486 totalFile.write("</table>\n")
488 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
491 #Write results to a raw text file as well
492 textResults = open(resultsDirectory + "total.txt", "w")
494 textResults.write(agent["name"] + " " + str(agent["totalScore"]) + "\n")
501 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."