b44012b0397952591f017c6bba41cae8f9c1df3f
[progcomp2012.git] / judge / simulator / simulate.py
1 #!/usr/bin/python -u
2
3 '''
4  simulate.py - simulation script for the 2012 UCC Programming Competition
5         NOTE: This is not the manager program for a stratego game
6         It merely calls the manager program as appropriate, and records results
7         Plays exactly ONE round, but does not overwrite previously played rounds
8         eg: run once to generate round1.results, twice to generate round2.results etc
9         Also generates total.scores based on results from every round.
10
11         Now (sortof) generates .html files to display results in a prettiful manner.
12         
13
14  author Sam Moore (matches) [SZM]
15  website http://matches.ucc.asn.au/stratego
16  email [email protected] or [email protected]
17  git git.ucc.asn.au/progcomp2012.git
18 '''
19
20 import os
21 import sys
22 from time import time
23
24 #Global variables/arguments
25
26 baseDirectory = "../.." #Base directory for results, logs, agents
27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
28 nRounds = 1
29
30 timeoutValue = 2
31
32 if len(sys.argv) >= 2:
33         nRounds = int(sys.argv[1])
34 if len(sys.argv) >= 3:
35         nGames = int(sys.argv[2])
36         if nGames % 2 != 0:
37                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
38 if len(sys.argv) >= 4:
39         baseDirectory = sys.argv[3]
40 if len(sys.argv) >= 6:
41         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
42         sys.exit(1)
43
44 resultsDirectory = baseDirectory+"/web/results/" #Where results will go (results are in the form of text files of agent names and scores)
45 logDirectory = baseDirectory+"/web/log/" #Where log files go (direct output of manager program)
46 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
47 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
48 if len(sys.argv) >= 5:
49         managerPath = sys.argv[5] 
50
51
52 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
53 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
54
55
56 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
57 verbose = True
58
59
60
61 #Check the manager program exists TODO: And is executable!
62 if os.path.exists(managerPath) == False:
63         print "Manager program at \""+managerPath+"\" doesn't exist!"
64         sys.exit(1)
65
66 #Make necessary directories
67
68 if os.path.exists(resultsDirectory) == False:
69         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
70
71         
72 #Identify the round number by reading from the "info" file in the results directory, if it doesn't exist then start at round 1.
73 if os.path.exists(resultsDirectory+"info") == False:
74         totalRounds = 1
75 else:
76         info = open(resultsDirectory+"info", "r")
77         totalRounds = int(info.readline().strip())
78         info.close()
79         os.remove(resultsDirectory+"info")
80
81 info = open(resultsDirectory+"info", "w")
82 info.write(str(totalRounds + nRounds) + "\n")
83 info.close()
84         
85
86
87 if os.path.exists(logDirectory) == False:
88         os.mkdir(logDirectory) #Make the log directory if it didn't exist
89
90
91 startTime = time() #Record time at which simulation starts
92
93 if verbose:
94         if nRounds > 1:
95                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
96         else:
97                 print "Simulating one round."
98         print ""
99         print "Identifying possible agents in \""+agentsDirectory+"\""
100
101 #Get all agent names from agentsDirectory
102 agentNames = os.listdir(agentsDirectory) 
103 agents = []
104 for name in agentNames:
105         if verbose:
106                 sys.stdout.write("Scan \""+name+"\"... ")
107         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
108                 if verbose:
109                         sys.stdout.write(" Invalid! (Not a directory)\n")
110                 continue
111
112         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist   
113                 if verbose:
114                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
115                 continue
116
117         infoFile = open(agentsDirectory+name+"/info", "r")
118         agentExecutable = agentsDirectory+name+"/"+(infoFile.readline().strip())
119         author = infoFile.readline().strip()
120         language = infoFile.readline().strip()
121         description = ""
122         while True:
123                 line = infoFile.readline()
124                 if len(line) > 0:
125                         description += line
126                 else:
127                         break
128         infoFile.close()
129         
130         if os.path.exists(agentExecutable) == False:
131                 if verbose:
132                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
133                 continue
134
135
136         if verbose:
137                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
138
139         #Convert array of valid names into array of dictionaries containing information about each agent
140         #I'm starting to like python...
141         agents.append({"name":name, "path":agentExecutable, "author":author, "language":language, "description":description, "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
142
143 if len(agents) == 0:
144         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
145         sys.exit(0)
146 if verbose:
147         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
148         print ""
149
150 #Prepare the pretty .html files if they don't exist
151 if verbose:
152         print "Preparing .html results files..."
153
154
155 for agent in agents:
156         if os.path.exists(resultsDirectory+agent["name"] + ".html") == False:
157                 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
158                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " overview</title>\n</head>\n<body>\n<h1> Overview for " + agent["name"]+" </h1>\n")
159                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
160                 agentFile.write("<tr> <th> Name </th> <th> Author </th> <th> Language </th> </tr>\n")
161                 agentFile.write("<tr> <td> "+agent["name"]+" </td> <td> "+agent["author"]+" </td> <td> "+agent["language"]+" </td> </tr>\n")
162                 agentFile.write("</table>\n");
163
164                 agentFile.write("<p> <b>Description</b> </p>\n")
165                 agentFile.write("<p> " + agent["description"] + " </p>\n")
166                 agentFile.close()
167
168         os.rename(resultsDirectory+agent["name"] + ".html", "tmpfile")
169         
170         oldFile = open("tmpfile", "r")
171         agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
172         line = oldFile.readline()
173         while line != "":
174                 #if verbose:
175                 #       print "Interpreting line \"" + line.strip() + "\""
176                 if line.strip() == "</body>":
177                         break
178                 elif line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
179                         agentFile.write(line)
180                         line = oldFile.readline()
181                         
182                         values = line.split(' ')
183                         agent["totalScore"] += int(values[2].strip())
184                         agent["Wins"] += int(values[5].strip())
185                         agent["Losses"] += int(values[8].strip())
186                         agent["Draws"] += int(values[11].strip())
187                         agent["Illegal"] += int(values[14].strip())
188                         agent["Errors"] += int(values[17].strip())
189                 agentFile.write(line)
190                 line = oldFile.readline()
191
192         if verbose:
193                 print "Prepared results file \"" + resultsDirectory+agent["name"] + ".html\"."
194         oldFile.close()
195         agentFile.close()
196         os.remove("tmpfile")
197
198 if verbose:
199         print ""
200
201 #Do each round...
202 totalGames = nGames/2 * len(agents) * (len(agents)-1)
203 for roundNumber in range(totalRounds, totalRounds + nRounds):
204
205         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
206                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
207
208         for agent in agents:
209                 agent.update({"name":agent["name"], "path":agent["path"],  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
210
211         
212         print "Commencing ROUND " + str(roundNumber) + " combat!"
213         print "Total: " + str(totalGames) + " games to be played. This could take a while... (Estimate 60s/game)"
214
215
216
217         managerErrors = 0
218         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
219         gameNumber = 0
220         for red in agents:  #for each agent playing as red,
221                 for blue in agents: #against each other agent, playing as blue
222                         if red == blue:
223                                 continue #Exclude battles against self
224                         gameNumber += 1
225                         gameID = str(roundNumber) + "." + str(gameNumber)
226                         for i in range(1, nGames/2 + 1):
227                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
228                                 if verbose:
229                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
230                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(gameID)
231                                 outline = os.popen(managerPath + " -o " + logFile + " -T " + str(timeoutValue) + " " + red["path"] + " " + blue["path"], "r").read()
232                                 results = outline.split(' ')
233                         
234                                 if len(results) != 6:
235                                         if verbose:
236                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
237                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
238                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
239                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
240                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
241                                         managerErrors += 1
242                                 else:
243
244                                         if results[1] == "RED":
245                                                 endColour = red
246                                                 otherColour = blue
247                                                 endStr = "RED"
248                                                 otherStr = "BLUE"
249                                         elif results[1] == "BLUE":
250                                                 endColour = blue
251                                                 otherColour = red
252                                                 endStr = "BLUE"
253                                                 otherStr = "RED"
254
255
256                                         if results[1] == "BOTH":
257                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
258                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
259                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
260                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
261                                                 managerErrors += 1
262                                         else:
263                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
264                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
265                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
266                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
267                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
268                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
269
270                                         
271                                         if verbose:
272                                                 sys.stdout.write(" Result \"")
273                                                 for ii in range(1, len(results)):
274                                                         sys.stdout.write(results[ii].strip())
275                                                         if ii < (len(results) - 1):
276                                                                 sys.stdout.write(" ")
277                                                 sys.stdout.write("\"\n")
278                 
279         if verbose:
280                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
281         if managerErrors != 0:
282                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
283
284         if verbose:
285                 print "" 
286         #We should now have complete score values.
287                 
288         '''
289                 Obselete, non prettified results
290         if verbose:
291                 sys.stdout.write("Creating raw results files for ROUND " + str(roundNumber) + "... ")
292
293         agents.sort(key = lambda e : e["score"], reverse=True) #Sort the agents based on score
294         
295         resultsFile = open(resultsDirectory+"round"+str(roundNumber)+".results", "w") #Create a file to store all the scores for this round
296         for agent in agents:
297                 resultsFile.write(agent["name"] + " " + str(agent["score"]) +"\n") #Write the agent names and scores into the file, in descending order
298
299         if verbose:
300                 sys.stdout.write(" Complete!\n")
301                 sys.stdout.write("Updating total scores... ");
302         
303         #Now update the total scores
304         if os.path.exists(resultsDirectory+"total.scores"):
305                 if verbose:
306                         sys.stdout.write(" Reading from \""+resultsDirectory+"total.scores\" to update scores... ")
307                 totalFile = open(resultsDirectory+"total.scores", "r") #Try to open the total.scores file
308                 for line in totalFile: #For all entries, 
309                         data = line.split(' ')
310                         for agent in agents:
311                                 if agent["name"] == data[0]:
312                                         agent["totalScore"] = int(data[1]) + agent["score"][0] #Simply increment the current score by the recorded total score of the matching file entry
313                                         break
314                 totalFile.close() #Close the file, so we can delete it
315                 os.remove(resultsDirectory+"total.scores") #Delete the file
316                 #Sort the agents again
317                 agents.sort(key = lambda e : e["totalScore"], reverse=True)
318
319         else:
320                 if verbose:
321                         sys.stdout.write(" First round - creating \""+resultsDirectory+"total.scores\"... ")
322         if verbose:
323                 sys.stdout.write(" Complete!\n")
324                 print "Finished writing results for ROUND " + str(roundNumber)
325                 print ""
326         '''
327         if verbose:     
328                 print "RESULTS FOR ROUND " + str(roundNumber)
329
330         #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
331                 for agent in agents:    
332                 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
333                 #if verbose:
334                                 print "Agent: " + str(agent)
335         
336
337         if verbose:
338                 print "Updating pretty .html files... "
339
340         for agent in agents:
341                 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
342                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
343                 agentFile.write("<h3> Round Overview </h3>\n")
344                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
345                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
346                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
347
348                 agentFile.write("</table>\n")
349                 agentFile.write("<p> <a href=round"+str(roundNumber)+".html>Round "+str(roundNumber) + " Scoreboard</a></p>\n")
350
351                 agentFile.write("<h3> Detailed </h3>\n")
352                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
353                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
354                 
355                 
356
357                 for index in range(0, len(agent["ALL"])):
358                         if agent["ALL"][index][4] == "RED":
359                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+agent["name"]+".vs."+agent["ALL"][index][0]+"."+str(agent["ALL"][index][1])
360                         else:
361                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+agent["ALL"][index][0]+".vs."+agent["name"]+"."+str(agent["ALL"][index][1])
362                         agentFile.write("<tr> <td> <a href="+logFile+">" + str(agent["ALL"][index][1]) + " </a> </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
363                 agentFile.write("</table>\n")
364                 
365                 agent["totalScore"] += agent["score"][0]
366                 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
367                 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
368                 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
369                 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
370                 agent["Errors"] += len(agent["INTERNAL_ERROR"])
371
372                 agentFile.write("<h3> Accumulated Results </h3>\n")
373                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
374                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
375                 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
376
377                 agentFile.write("</table>\n")
378
379
380                 agentFile.close()       
381
382         #Update round file
383         roundFile = open(resultsDirectory + "round"+str(roundNumber)+".html", "w")
384         roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
385         roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
386         roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
387         roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
388         agents.sort(key = lambda e : e["score"][0], reverse=True)
389         for agent in agents:
390                 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
391         roundFile.write("</table>\n")
392         roundFile.write("<p> <a href=total.html>Current Scoreboard</a></p>\n")
393         roundFile.write("</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
394         roundFile.close()
395
396
397         
398         
399
400 if verbose:
401         print "Finalising .html files... "
402 for agent in agents:
403         agentFile = open(resultsDirectory + agent["name"]+".html", "a")
404
405         #Write the "total" statistics
406
407         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
408         agentFile.close()
409
410         if os.path.exists(resultsDirectory + "total.html") == True:
411                 os.remove(resultsDirectory + "total.html") #Delete the file
412
413 totalFile = open(resultsDirectory + "total.html", "w")
414 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
415 totalFile.write("<h1> Total Overview </h1>\n")
416 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
417 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
418 agents.sort(key = lambda e : e["totalScore"], reverse=True)
419 for agent in agents:
420         totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
421 totalFile.write("</table>\n")
422
423 totalFile.write("<h2> Round Summaries </h2>\n")
424 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
425 for i in range(1, totalRounds+1):
426         totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
427 totalFile.write("</table>\n")
428
429 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
430 totalFile.close()
431
432         
433 if verbose:
434         print "Done!"
435
436 endTime = time()
437 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
438 sys.exit(0)

UCC git Repository :: git.ucc.asn.au