Tweaking manager/simulate.py, updating manual and webpage
[progcomp2012.git] / judge / simulator / simulate.py
1 #!/usr/bin/python -u
2
3 '''
4  simulate.py - simulation script for the 2012 UCC Programming Competition
5         NOTE: This is not the manager program for a stratego game
6         It merely calls the manager program as appropriate, and records results
7         Plays exactly ONE round, but does not overwrite previously played rounds
8         eg: run once to generate round1.results, twice to generate round2.results etc
9         Also generates total.scores based on results from every round.
10
11         Now (sortof) generates .html files to display results in a prettiful manner.
12         
13
14  author Sam Moore (matches) [SZM]
15  website http://matches.ucc.asn.au/stratego
16  email progcomp@ucc.asn.au or matches@ucc.asn.au
17  git git.ucc.asn.au/progcomp2012.git
18 '''
19
20 import os
21 import sys
22 from time import time
23
24 #Global variables/arguments
25
26 baseDirectory = "../.." #Base directory for results, logs, agents
27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
28 nRounds = 1
29
30 timeoutValue = 2
31
32 if len(sys.argv) >= 2:
33         nRounds = int(sys.argv[1])
34 if len(sys.argv) >= 3:
35         nGames = int(sys.argv[2])
36         if nGames % 2 != 0:
37                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
38 if len(sys.argv) >= 4:
39         baseDirectory = sys.argv[3]
40 if len(sys.argv) >= 6:
41         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
42         sys.exit(1)
43
44 resultsDirectory = baseDirectory+"/web/results/" #Where results will go (results are in the form of text files of agent names and scores)
45 logDirectory = baseDirectory+"/web/log/" #Where log files go (direct output of manager program)
46 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
47 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
48 if len(sys.argv) >= 5:
49         managerPath = sys.argv[5] 
50
51
52 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
53 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
54
55
56 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
57 verbose = True
58
59
60
61 #Check the manager program exists TODO: And is executable!
62 if os.path.exists(managerPath) == False:
63         print "Manager program at \""+managerPath+"\" doesn't exist!"
64         sys.exit(1)
65
66 #Make necessary directories
67
68 if os.path.exists(resultsDirectory) == False:
69         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
70
71         
72 #Identify the round number by reading from the "info" file in the results directory, if it doesn't exist then start at round 1.
73 if os.path.exists(resultsDirectory+"info") == False:
74         totalRounds = 1
75 else:
76         info = open(resultsDirectory+"info", "r")
77         totalRounds = int(info.readline().strip())
78         info.close()
79         os.remove(resultsDirectory+"info")
80
81 info = open(resultsDirectory+"info", "w")
82 info.write(str(totalRounds + nRounds) + "\n")
83 info.close()
84         
85
86
87 if os.path.exists(logDirectory) == False:
88         os.mkdir(logDirectory) #Make the log directory if it didn't exist
89
90
91 startTime = time() #Record time at which simulation starts
92
93 if verbose:
94         if nRounds > 1:
95                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
96         else:
97                 print "Simulating one round."
98         print ""
99         print "Identifying possible agents in \""+agentsDirectory+"\""
100
101 #Get all agent names from agentsDirectory
102 agentNames = os.listdir(agentsDirectory) 
103 agents = []
104 for name in agentNames:
105         if verbose:
106                 sys.stdout.write("Scan \""+name+"\"... ")
107         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
108                 if verbose:
109                         sys.stdout.write(" Invalid! (Not a directory)\n")
110                 continue
111
112         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist   
113                 if verbose:
114                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
115                 continue
116
117         infoFile = open(agentsDirectory+name+"/info", "r")
118         agentExecutable = agentsDirectory+name+"/"+(infoFile.readline().strip())
119         author = infoFile.readline().strip()
120         language = infoFile.readline().strip()
121         description = ""
122         while True:
123                 line = infoFile.readline()
124                 if len(line) > 0:
125                         description += line
126                 else:
127                         break
128         infoFile.close()
129         
130         if os.path.exists(agentExecutable.split(" ")[0]) == False:
131                 if verbose:
132                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
133                 continue
134
135
136         if verbose:
137                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
138
139         #Convert array of valid names into array of dictionaries containing information about each agent
140         #I'm starting to like python...
141         agents.append({"name":name, "path":agentExecutable, "author":author, "language":language, "description":description, "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
142
143 if len(agents) == 0:
144         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
145         sys.exit(0)
146 if verbose:
147         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
148         print ""
149
150 #Prepare the pretty .html files if they don't exist
151 if verbose:
152         print "Preparing .html results files..."
153
154
155 for agent in agents:
156         if os.path.exists(resultsDirectory+agent["name"] + ".html") == False:
157                 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
158                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " overview</title>\n</head>\n<body>\n<h1> Overview for " + agent["name"]+" </h1>\n")
159                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
160                 agentFile.write("<tr> <th> Name </th> <th> Author </th> <th> Language </th> </tr>\n")
161                 agentFile.write("<tr> <td> "+agent["name"]+" </td> <td> "+agent["author"]+" </td> <td> "+agent["language"]+" </td> </tr>\n")
162                 agentFile.write("</table>\n");
163
164                 agentFile.write("<p> <b>Description</b> </p>\n")
165                 agentFile.write("<p> " + agent["description"] + " </p>\n")
166                 agentFile.close()
167
168         os.rename(resultsDirectory+agent["name"] + ".html", "tmpfile")
169         
170         oldFile = open("tmpfile", "r")
171         agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
172         line = oldFile.readline()
173         while line != "":
174                 #if verbose:
175                 #       print "Interpreting line \"" + line.strip() + "\""
176                 if line.strip() == "</body>":
177                         break
178                 elif line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
179                         agentFile.write(line)
180                         line = oldFile.readline()
181                         
182                         values = line.split(' ')
183                         agent["totalScore"] += int(values[2].strip())
184                         agent["Wins"] += int(values[5].strip())
185                         agent["Losses"] += int(values[8].strip())
186                         agent["Draws"] += int(values[11].strip())
187                         agent["Illegal"] += int(values[14].strip())
188                         agent["Errors"] += int(values[17].strip())
189                 agentFile.write(line)
190                 line = oldFile.readline()
191
192         if verbose:
193                 print "Prepared results file \"" + resultsDirectory+agent["name"] + ".html\"."
194         oldFile.close()
195         agentFile.close()
196         os.remove("tmpfile")
197
198 if verbose:
199         print ""
200
201 #Do each round...
202 totalGames = nGames/2 * len(agents) * (len(agents)-1)
203 for roundNumber in range(totalRounds, totalRounds + nRounds):
204
205         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
206                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
207
208         for agent in agents:
209                 agent.update({"name":agent["name"], "path":agent["path"],  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
210
211         
212         print "Commencing ROUND " + str(roundNumber) + " combat!"
213         print "Total: " + str(totalGames) + " games to be played. This could take a while..."
214
215
216
217         managerErrors = 0
218         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
219         gameNumber = 0
220         for red in agents:  #for each agent playing as red,
221                 for blue in agents: #against each other agent, playing as blue
222                         if red == blue:
223                                 continue #Exclude battles against self
224                         
225                         
226                         for i in range(1, nGames/2 + 1):
227                                 gameNumber += 1
228                                 gameID = str(roundNumber) + "." + str(gameNumber)
229                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
230                                 if verbose:
231                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
232                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(gameID)
233                                 errorLog = [logDirectory + "error/" + red["name"] + "."+str(gameID), logDirectory + "error/" + blue["name"] + "."+str(gameID)]
234                                 #Run the game, outputting to logFile; stderr of (both) AI programs is directed to logFile.stderr
235                                 outline = os.popen(managerPath + " -o " + logFile + " -T " + str(timeoutValue) + " \"" + red["path"] + "\" \"" + blue["path"] + "\" 2>> " + logFile+".stderr", "r").read()
236                                 
237                                 #If there were no errors, get rid of the stderr file
238                                 if os.stat(logFile+".stderr").st_size <= 0:
239                                         os.remove(logFile+".stderr")
240                                 results = outline.split(' ')
241                         
242                                 if len(results) != 6:
243                                         if verbose:
244                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
245                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
246                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
247                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
248                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
249                                         managerErrors += 1
250                                 else:
251
252                                         if results[1] == "RED":
253                                                 endColour = red
254                                                 otherColour = blue
255                                                 endStr = "RED"
256                                                 otherStr = "BLUE"
257                                         elif results[1] == "BLUE":
258                                                 endColour = blue
259                                                 otherColour = red
260                                                 endStr = "BLUE"
261                                                 otherStr = "RED"
262
263
264                                         if results[1] == "BOTH":
265                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
266                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
267                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
268                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
269                                                 managerErrors += 1
270                                         else:
271                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
272                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
273                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
274                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
275                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
276                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
277
278                                         
279                                         if verbose:
280                                                 sys.stdout.write(" Result \"")
281                                                 for ii in range(1, len(results)):
282                                                         sys.stdout.write(results[ii].strip())
283                                                         if ii < (len(results) - 1):
284                                                                 sys.stdout.write(" ")
285                                                 sys.stdout.write("\"\n")
286                 
287         if verbose:
288                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
289         if managerErrors != 0:
290                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
291
292         if verbose:
293                 print "" 
294         #We should now have complete score values.
295         if verbose:     
296                 print "RESULTS FOR ROUND " + str(roundNumber)
297
298         #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
299                 #for agent in agents:   
300                 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
301                 #if verbose:
302                 #               print "Agent: " + str(agent)
303         
304
305         if verbose:
306                 print "Updating pretty .html files... "
307
308         for agent in agents:
309                 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
310                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
311                 agentFile.write("<h3> Round Overview </h3>\n")
312                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
313                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
314                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
315
316                 agentFile.write("</table>\n")
317                 agentFile.write("<p> <a href=round"+str(roundNumber)+".html>Round "+str(roundNumber) + " Scoreboard</a></p>\n")
318
319                 agentFile.write("<h3> Detailed </h3>\n")
320                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
321                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
322                 
323                 
324
325                 for index in range(0, len(agent["ALL"])):
326                         if agent["ALL"][index][4] == "RED":
327                                 logFile = "../log/round"+str(roundNumber) + "/"+agent["name"]+".vs."+agent["ALL"][index][0]+"."+str(agent["ALL"][index][1])
328                         else:
329                                 logFile = "../log/round"+str(roundNumber) + "/"+agent["ALL"][index][0]+".vs."+agent["name"]+"."+str(agent["ALL"][index][1])
330                         agentFile.write("<tr> <td> <a href="+logFile+">" + str(agent["ALL"][index][1]) + " </a> </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
331                 agentFile.write("</table>\n")
332                 
333                 agent["totalScore"] += agent["score"][0]
334                 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
335                 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
336                 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
337                 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
338                 agent["Errors"] += len(agent["INTERNAL_ERROR"])
339
340                 agentFile.write("<h3> Accumulated Results </h3>\n")
341                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
342                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
343                 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
344
345                 agentFile.write("</table>\n")
346
347
348                 agentFile.close()       
349
350         #Update round file
351         roundFile = open(resultsDirectory + "round"+str(roundNumber)+".html", "w")
352         roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
353         roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
354         roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
355         roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
356         agents.sort(key = lambda e : e["score"][0], reverse=True)
357         for agent in agents:
358                 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
359         roundFile.write("</table>\n")
360         roundFile.write("<p> <a href=total.html>Current Scoreboard</a></p>\n")
361         roundFile.write("</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
362         roundFile.close()
363
364
365         
366         
367
368 if verbose:
369         print "Finalising .html files... "
370 for agent in agents:
371         agentFile = open(resultsDirectory + agent["name"]+".html", "a")
372
373         #Write the "total" statistics
374
375         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
376         agentFile.close()
377
378         if os.path.exists(resultsDirectory + "total.html") == True:
379                 os.remove(resultsDirectory + "total.html") #Delete the file
380
381 totalFile = open(resultsDirectory + "total.html", "w")
382 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
383 totalFile.write("<h1> Total Overview </h1>\n")
384 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
385 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
386 agents.sort(key = lambda e : e["totalScore"], reverse=True)
387 for agent in agents:
388         totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
389 totalFile.write("</table>\n")
390
391 totalFile.write("<h2> Round Summaries </h2>\n")
392 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
393 for i in range(1, totalRounds+1):
394         totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
395 totalFile.write("</table>\n")
396
397 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
398 totalFile.close()
399
400         
401 if verbose:
402         print "Done!"
403
404 endTime = time()
405 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
406 sys.exit(0)

UCC git Repository :: git.ucc.asn.au