Fixed bug in simulate.py + hacky fix for bug in vixen
[progcomp2012.git] / judge / simulator / simulate.py
1 #!/usr/bin/python -u
2
3 '''
4  simulate.py - simulation script for the 2012 UCC Programming Competition
5         NOTE: This is not the manager program for a stratego game
6         It merely calls the manager program as appropriate, and records results
7         Plays exactly ONE round, but does not overwrite previously played rounds
8         eg: run once to generate round1.results, twice to generate round2.results etc
9         Also generates total.scores based on results from every round.
10
11         Now (sortof) generates .html files to display results in a prettiful manner.
12         
13
14  author Sam Moore (matches) [SZM]
15  website http://matches.ucc.asn.au/stratego
16  email progcomp@ucc.asn.au or matches@ucc.asn.au
17  git git.ucc.asn.au/progcomp2012.git
18 '''
19
20 import os
21 import sys
22 from time import time
23
24 #Global variables/arguments
25
26 baseDirectory = "../.." #Base directory for results, logs, agents
27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
28 nRounds = 1
29
30 timeoutValue = 2
31
32 if len(sys.argv) >= 2:
33         nRounds = int(sys.argv[1])
34 if len(sys.argv) >= 3:
35         nGames = int(sys.argv[2])
36         if nGames % 2 != 0:
37                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
38 if len(sys.argv) >= 4:
39         baseDirectory = sys.argv[3]
40 if len(sys.argv) >= 6:
41         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
42         sys.exit(1)
43
44 resultsDirectory = baseDirectory+"/web/results/" #Where results will go (results are in the form of text files of agent names and scores)
45 logDirectory = baseDirectory+"/web/log/" #Where log files go (direct output of manager program)
46 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
47 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
48 if len(sys.argv) >= 5:
49         managerPath = sys.argv[5] 
50
51
52 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
53 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
54
55
56 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
57 verbose = True
58
59
60
61 #Check the manager program exists TODO: And is executable!
62 if os.path.exists(managerPath) == False:
63         print "Manager program at \""+managerPath+"\" doesn't exist!"
64         sys.exit(1)
65
66 #Make necessary directories
67
68 if os.path.exists(resultsDirectory) == False:
69         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
70
71         
72 #Identify the round number by reading from the "info" file in the results directory, if it doesn't exist then start at round 1.
73 if os.path.exists(resultsDirectory+"info") == False:
74         totalRounds = 1
75 else:
76         info = open(resultsDirectory+"info", "r")
77         totalRounds = int(info.readline().strip())
78         info.close()
79         os.remove(resultsDirectory+"info")
80
81 info = open(resultsDirectory+"info", "w")
82 info.write(str(totalRounds + nRounds) + "\n")
83 info.close()
84         
85
86
87 if os.path.exists(logDirectory) == False:
88         os.mkdir(logDirectory) #Make the log directory if it didn't exist
89
90
91 startTime = time() #Record time at which simulation starts
92
93 if verbose:
94         if nRounds > 1:
95                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
96         else:
97                 print "Simulating one round."
98         print ""
99         print "Identifying possible agents in \""+agentsDirectory+"\""
100
101 #Get all agent names from agentsDirectory
102 agentNames = os.listdir(agentsDirectory) 
103 agents = []
104 for name in agentNames:
105         if verbose:
106                 sys.stdout.write("Scan \""+name+"\"... ")
107         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
108                 if verbose:
109                         sys.stdout.write(" Invalid! (Not a directory)\n")
110                 continue
111
112         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist   
113                 if verbose:
114                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
115                 continue
116
117         infoFile = open(agentsDirectory+name+"/info", "r")
118         agentExecutable = agentsDirectory+name+"/"+(infoFile.readline().strip())
119         author = infoFile.readline().strip()
120         language = infoFile.readline().strip()
121         description = ""
122         while True:
123                 line = infoFile.readline()
124                 if len(line) > 0:
125                         description += line
126                 else:
127                         break
128         infoFile.close()
129         
130         if os.path.exists(agentExecutable) == False:
131                 if verbose:
132                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
133                 continue
134
135
136         if verbose:
137                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
138
139         #Convert array of valid names into array of dictionaries containing information about each agent
140         #I'm starting to like python...
141         agents.append({"name":name, "path":agentExecutable, "author":author, "language":language, "description":description, "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
142
143 if len(agents) == 0:
144         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
145         sys.exit(0)
146 if verbose:
147         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
148         print ""
149
150 #Prepare the pretty .html files if they don't exist
151 if verbose:
152         print "Preparing .html results files..."
153
154
155 for agent in agents:
156         if os.path.exists(resultsDirectory+agent["name"] + ".html") == False:
157                 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
158                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " overview</title>\n</head>\n<body>\n<h1> Overview for " + agent["name"]+" </h1>\n")
159                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
160                 agentFile.write("<tr> <th> Name </th> <th> Author </th> <th> Language </th> </tr>\n")
161                 agentFile.write("<tr> <td> "+agent["name"]+" </td> <td> "+agent["author"]+" </td> <td> "+agent["language"]+" </td> </tr>\n")
162                 agentFile.write("</table>\n");
163
164                 agentFile.write("<p> <b>Description</b> </p>\n")
165                 agentFile.write("<p> " + agent["description"] + " </p>\n")
166                 agentFile.close()
167
168         os.rename(resultsDirectory+agent["name"] + ".html", "tmpfile")
169         
170         oldFile = open("tmpfile", "r")
171         agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
172         line = oldFile.readline()
173         while line != "":
174                 #if verbose:
175                 #       print "Interpreting line \"" + line.strip() + "\""
176                 if line.strip() == "</body>":
177                         break
178                 elif line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
179                         agentFile.write(line)
180                         line = oldFile.readline()
181                         
182                         values = line.split(' ')
183                         agent["totalScore"] += int(values[2].strip())
184                         agent["Wins"] += int(values[5].strip())
185                         agent["Losses"] += int(values[8].strip())
186                         agent["Draws"] += int(values[11].strip())
187                         agent["Illegal"] += int(values[14].strip())
188                         agent["Errors"] += int(values[17].strip())
189                 agentFile.write(line)
190                 line = oldFile.readline()
191
192         if verbose:
193                 print "Prepared results file \"" + resultsDirectory+agent["name"] + ".html\"."
194         oldFile.close()
195         agentFile.close()
196         os.remove("tmpfile")
197
198 if verbose:
199         print ""
200
201 #Do each round...
202 totalGames = nGames/2 * len(agents) * (len(agents)-1)
203 for roundNumber in range(totalRounds, totalRounds + nRounds):
204
205         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
206                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
207
208         for agent in agents:
209                 agent.update({"name":agent["name"], "path":agent["path"],  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
210
211         
212         print "Commencing ROUND " + str(roundNumber) + " combat!"
213         print "Total: " + str(totalGames) + " games to be played. This could take a while... (Estimate 60s/game)"
214
215
216
217         managerErrors = 0
218         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
219         gameNumber = 0
220         for red in agents:  #for each agent playing as red,
221                 for blue in agents: #against each other agent, playing as blue
222                         if red == blue:
223                                 continue #Exclude battles against self
224                         
225                         
226                         for i in range(1, nGames/2 + 1):
227                                 gameNumber += 1
228                                 gameID = str(roundNumber) + "." + str(gameNumber)
229                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
230                                 if verbose:
231                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
232                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(gameID)
233                                 errorLog = [logDirectory + "error/" + red["name"] + "."+str(gameID), logDirectory + "error/" + blue["name"] + "."+str(gameID)]
234                                 outline = os.popen(managerPath + " -o " + logFile + " -T " + str(timeoutValue) + " " + red["path"] + " " + blue["path"], "r").read()
235                                 results = outline.split(' ')
236                         
237                                 if len(results) != 6:
238                                         if verbose:
239                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
240                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
241                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
242                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
243                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
244                                         managerErrors += 1
245                                 else:
246
247                                         if results[1] == "RED":
248                                                 endColour = red
249                                                 otherColour = blue
250                                                 endStr = "RED"
251                                                 otherStr = "BLUE"
252                                         elif results[1] == "BLUE":
253                                                 endColour = blue
254                                                 otherColour = red
255                                                 endStr = "BLUE"
256                                                 otherStr = "RED"
257
258
259                                         if results[1] == "BOTH":
260                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
261                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
262                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
263                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
264                                                 managerErrors += 1
265                                         else:
266                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
267                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
268                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
269                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
270                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
271                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
272
273                                         
274                                         if verbose:
275                                                 sys.stdout.write(" Result \"")
276                                                 for ii in range(1, len(results)):
277                                                         sys.stdout.write(results[ii].strip())
278                                                         if ii < (len(results) - 1):
279                                                                 sys.stdout.write(" ")
280                                                 sys.stdout.write("\"\n")
281                 
282         if verbose:
283                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
284         if managerErrors != 0:
285                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
286
287         if verbose:
288                 print "" 
289         #We should now have complete score values.
290                 
291         '''
292                 Obselete, non prettified results
293         if verbose:
294                 sys.stdout.write("Creating raw results files for ROUND " + str(roundNumber) + "... ")
295
296         agents.sort(key = lambda e : e["score"], reverse=True) #Sort the agents based on score
297         
298         resultsFile = open(resultsDirectory+"round"+str(roundNumber)+".results", "w") #Create a file to store all the scores for this round
299         for agent in agents:
300                 resultsFile.write(agent["name"] + " " + str(agent["score"]) +"\n") #Write the agent names and scores into the file, in descending order
301
302         if verbose:
303                 sys.stdout.write(" Complete!\n")
304                 sys.stdout.write("Updating total scores... ");
305         
306         #Now update the total scores
307         if os.path.exists(resultsDirectory+"total.scores"):
308                 if verbose:
309                         sys.stdout.write(" Reading from \""+resultsDirectory+"total.scores\" to update scores... ")
310                 totalFile = open(resultsDirectory+"total.scores", "r") #Try to open the total.scores file
311                 for line in totalFile: #For all entries, 
312                         data = line.split(' ')
313                         for agent in agents:
314                                 if agent["name"] == data[0]:
315                                         agent["totalScore"] = int(data[1]) + agent["score"][0] #Simply increment the current score by the recorded total score of the matching file entry
316                                         break
317                 totalFile.close() #Close the file, so we can delete it
318                 os.remove(resultsDirectory+"total.scores") #Delete the file
319                 #Sort the agents again
320                 agents.sort(key = lambda e : e["totalScore"], reverse=True)
321
322         else:
323                 if verbose:
324                         sys.stdout.write(" First round - creating \""+resultsDirectory+"total.scores\"... ")
325         if verbose:
326                 sys.stdout.write(" Complete!\n")
327                 print "Finished writing results for ROUND " + str(roundNumber)
328                 print ""
329         '''
330         if verbose:     
331                 print "RESULTS FOR ROUND " + str(roundNumber)
332
333         #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
334                 #for agent in agents:   
335                 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
336                 #if verbose:
337                 #               print "Agent: " + str(agent)
338         
339
340         if verbose:
341                 print "Updating pretty .html files... "
342
343         for agent in agents:
344                 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
345                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
346                 agentFile.write("<h3> Round Overview </h3>\n")
347                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
348                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
349                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
350
351                 agentFile.write("</table>\n")
352                 agentFile.write("<p> <a href=round"+str(roundNumber)+".html>Round "+str(roundNumber) + " Scoreboard</a></p>\n")
353
354                 agentFile.write("<h3> Detailed </h3>\n")
355                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
356                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
357                 
358                 
359
360                 for index in range(0, len(agent["ALL"])):
361                         if agent["ALL"][index][4] == "RED":
362                                 logFile = "log/round"+str(roundNumber) + "/"+agent["name"]+".vs."+agent["ALL"][index][0]+"."+str(agent["ALL"][index][1])
363                         else:
364                                 logFile = "log/round"+str(roundNumber) + "/"+agent["ALL"][index][0]+".vs."+agent["name"]+"."+str(agent["ALL"][index][1])
365                         agentFile.write("<tr> <td> <a href="+logFile+">" + str(agent["ALL"][index][1]) + " </a> </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
366                 agentFile.write("</table>\n")
367                 
368                 agent["totalScore"] += agent["score"][0]
369                 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
370                 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
371                 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
372                 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
373                 agent["Errors"] += len(agent["INTERNAL_ERROR"])
374
375                 agentFile.write("<h3> Accumulated Results </h3>\n")
376                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
377                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
378                 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
379
380                 agentFile.write("</table>\n")
381
382
383                 agentFile.close()       
384
385         #Update round file
386         roundFile = open(resultsDirectory + "round"+str(roundNumber)+".html", "w")
387         roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
388         roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
389         roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
390         roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
391         agents.sort(key = lambda e : e["score"][0], reverse=True)
392         for agent in agents:
393                 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
394         roundFile.write("</table>\n")
395         roundFile.write("<p> <a href=total.html>Current Scoreboard</a></p>\n")
396         roundFile.write("</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
397         roundFile.close()
398
399
400         
401         
402
403 if verbose:
404         print "Finalising .html files... "
405 for agent in agents:
406         agentFile = open(resultsDirectory + agent["name"]+".html", "a")
407
408         #Write the "total" statistics
409
410         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
411         agentFile.close()
412
413         if os.path.exists(resultsDirectory + "total.html") == True:
414                 os.remove(resultsDirectory + "total.html") #Delete the file
415
416 totalFile = open(resultsDirectory + "total.html", "w")
417 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
418 totalFile.write("<h1> Total Overview </h1>\n")
419 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
420 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
421 agents.sort(key = lambda e : e["totalScore"], reverse=True)
422 for agent in agents:
423         totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
424 totalFile.write("</table>\n")
425
426 totalFile.write("<h2> Round Summaries </h2>\n")
427 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
428 for i in range(1, totalRounds+1):
429         totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
430 totalFile.write("</table>\n")
431
432 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
433 totalFile.close()
434
435         
436 if verbose:
437         print "Done!"
438
439 endTime = time()
440 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
441 sys.exit(0)

UCC git Repository :: git.ucc.asn.au