76ca0a56d11b23145250bb806a5a98d306aa8cd5
[progcomp2012.git] / progcomp / judge / simulator / simulate.py
1 #!/usr/bin/python -u
2
3 '''
4  simulate.py - simulation script for the 2012 UCC Programming Competition
5         NOTE: This is not the manager program for a stratego game
6         It merely calls the manager program as appropriate, and records results
7         Plays exactly ONE round, but does not overwrite previously played rounds
8         eg: run once to generate round1.results, twice to generate round2.results etc
9         Also generates total.scores based on results from every round.
10
11         Now (sortof) generates .html files to display results in a prettiful manner.
12         
13
14  author Sam Moore (matches) [SZM]
15  website http://matches.ucc.asn.au/stratego
16  email progcomp@ucc.asn.au or matches@ucc.asn.au
17  git git.ucc.asn.au/progcomp2012.git
18 '''
19
20 import os
21 import sys
22 from time import time
23
24 #Global variables/arguments
25
26 baseDirectory = "../.." #Base directory for results, logs, agents
27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
28 nRounds = 1
29
30 if len(sys.argv) >= 2:
31         nRounds = int(sys.argv[1])
32 if len(sys.argv) >= 3:
33         nGames = int(sys.argv[2])
34         if nGames % 2 != 0:
35                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
36 if len(sys.argv) >= 4:
37         baseDirectory = sys.argv[3]
38 if len(sys.argv) >= 6:
39         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
40         sys.exit(1)
41
42 resultsDirectory = baseDirectory+"/results/" #Where results will go (results are in the form of text files of agent names and scores)
43 logDirectory = baseDirectory+"/log/" #Where log files go (direct output of manager program)
44 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
45 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
46 if len(sys.argv) >= 5:
47         managerPath = sys.argv[5] 
48
49
50 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
51 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
52
53
54 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
55 verbose = True
56
57
58
59 #Check the manager program exists TODO: And is executable!
60 if os.path.exists(managerPath) == False:
61         print "Manager program at \""+managerPath+"\" doesn't exist!"
62         sys.exit(1)
63
64 #Make necessary directories
65
66 if os.path.exists(resultsDirectory) == False:
67         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
68 #Identify the round number by reading the results directory
69 totalRounds = len(os.listdir(resultsDirectory)) + 1
70 if totalRounds > 1:
71         totalRounds -= 1
72
73 if os.path.exists(logDirectory) == False:
74         os.mkdir(logDirectory) #Make the log directory if it didn't exist
75
76
77 startTime = time() #Record time at which simulation starts
78
79 if verbose:
80         if nRounds > 1:
81                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
82         else:
83                 print "Simulating one round."
84         print ""
85         print "Identifying possible agents in \""+agentsDirectory+"\""
86
87 #Get all agent names from agentsDirectory
88 #TODO: Move this part outside the loop? It only has to happen once
89 agentNames = os.listdir(agentsDirectory) 
90 agents = []
91 for name in agentNames:
92         if verbose:
93                 sys.stdout.write("Scan \""+name+"\"... ")
94         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
95                 if verbose:
96                         sys.stdout.write(" Invalid! (Not a directory)\n")
97                 continue
98
99         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist   
100                 if verbose:
101                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
102                 continue
103
104         agentExecutable = agentsDirectory+name+"/"+(open(agentsDirectory+name+"/info").readline().strip())
105         
106         if os.path.exists(agentExecutable) == False:
107                 if verbose:
108                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
109                 continue
110
111
112         if verbose:
113                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
114
115         #Convert array of valid names into array of dictionaries containing information about each agent
116         #I'm starting to like python...
117         agents.append({"name":name, "path":agentExecutable,"score":[0], "totalScore":0, "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "INTERNAL_ERROR":[], "ALL":[]})     
118
119 if len(agents) == 0:
120         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
121         sys.exit(0)
122 if verbose:
123         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
124         print ""
125
126 #Prepare the pretty .html files if they don't exist
127 htmlDir = resultsDirectory + "pretty/"
128 if os.path.exists(htmlDir) == False:
129         os.mkdir(htmlDir)
130 if os.path.exists(htmlDir) == False:
131         print "Couldn't create directory \""+htmlDir+"\"."
132         sys.exit(1)
133
134 for agent in agents:
135         if os.path.exists(htmlDir+agent["name"] + ".html") == False:
136                 agentFile = open(htmlDir+agent["name"] + ".html", "w")
137                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " results</title>\n</head>\n<body>\n<h1> Results for " + agent["name"]+" </h1>\n</body>\n</html>\n")
138                 agentFile.close()
139
140         os.rename(htmlDir+agent["name"] + ".html", "tmpfile")
141         
142         oldFile = open("tmpfile")
143         agentFile = open(htmlDir+agent["name"] + ".html", "w")
144         for line in oldFile:
145                 if line.strip() == "</body>":
146                         break
147                 agentFile.write(line + "\n")
148         oldFile.close()
149         agentFile.close()
150         os.remove("tmpfile")
151
152 #Do each round...
153 totalGames = nGames/2 * len(agents) * (len(agents)-1)
154 for roundNumber in range(totalRounds, totalRounds + nRounds):
155
156         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
157                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
158
159         
160         print "Commencing ROUND " + str(roundNumber) + " combat!"
161         print "Total: " + str(totalGames) + " games to be played. This could take a while... (Estimate 60s/game)"
162
163
164
165         managerErrors = 0
166         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
167         gameNumber = 0
168         for red in agents:  #for each agent playing as red,
169                 for blue in agents: #against each other agent, playing as blue
170                         if red == blue:
171                                 continue #Exclude battles against self
172                         gameNumber += 1
173                         gameID = str(roundNumber) + "." + str(gameNumber)
174                         for i in range(1, nGames/2 + 1):
175                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
176                                 if verbose:
177                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
178                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(i)
179                                 outline = os.popen(managerPath + " -o " + logFile + " " + red["path"] + " " + blue["path"], "r").read()
180                                 results = outline.split(' ')
181                         
182                                 if len(results) != 6:
183                                         if verbose:
184                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
185                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
186                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
187                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
188                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
189                                         managerErrors += 1
190                                 else:
191
192                                         if results[1] == "RED":
193                                                 endColour = red
194                                                 otherColour = blue
195                                                 endStr = "RED"
196                                                 otherStr = "BLUE"
197                                         elif results[1] == "BLUE":
198                                                 endColour = blue
199                                                 otherColour = red
200                                                 endStr = "BLUE"
201                                                 otherStr = "RED"
202
203
204                                         if results[1] == "BOTH":
205                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
206                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
207                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
208                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
209                                                 managerErrors += 1
210                                         else:
211                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
212                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
213                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
214                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
215                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
216                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
217
218                                         
219                                         if verbose:
220                                                 sys.stdout.write(" Result \"")
221                                                 for ii in range(1, len(results)):
222                                                         sys.stdout.write(results[ii].strip())
223                                                         if ii < (len(results) - 1):
224                                                                 sys.stdout.write(" ")
225                                                 sys.stdout.write("\"\n")
226                 
227         if verbose:
228                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
229         if managerErrors != 0:
230                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
231
232         if verbose:
233                 print "" 
234         #We should now have complete score values.
235                 
236         if verbose:
237                 sys.stdout.write("Creating raw results files for ROUND " + str(roundNumber) + "... ")
238
239         agents.sort(key = lambda e : e["score"], reverse=True) #Sort the agents based on score
240         
241         resultsFile = open(resultsDirectory+"round"+str(roundNumber)+".results", "w") #Create a file to store all the scores for this round
242         for agent in agents:
243                 resultsFile.write(agent["name"] + " " + str(agent["score"]) +"\n") #Write the agent names and scores into the file, in descending order
244
245         if verbose:
246                 sys.stdout.write(" Complete!\n")
247                 sys.stdout.write("Updating total scores... ");
248         
249         #Now update the total scores
250         if os.path.exists(resultsDirectory+"total.scores"):
251                 if verbose:
252                         sys.stdout.write(" Reading from \""+resultsDirectory+"total.scores\" to update scores... ")
253                 totalFile = open(resultsDirectory+"total.scores", "r") #Try to open the total.scores file
254                 for line in totalFile: #For all entries, 
255                         data = line.split(' ')
256                         for agent in agents:
257                                 if agent["name"] == data[0]:
258                                         agent["totalScore"] = int(data[1]) + agent["score"][0] #Simply increment the current score by the recorded total score of the matching file entry
259                                         break
260                 totalFile.close() #Close the file, so we can delete it
261                 os.remove(resultsDirectory+"total.scores") #Delete the file
262                 #Sort the agents again
263                 agents.sort(key = lambda e : e["totalScore"], reverse=True)
264
265         else:
266                 if verbose:
267                         sys.stdout.write(" First round - creating \""+resultsDirectory+"total.scores\"... ")
268         if verbose:
269                 sys.stdout.write(" Complete!\n")
270                 print "Finished writing results for ROUND " + str(roundNumber)
271                 print ""
272         
273         if verbose:     
274                 print "RESULTS FOR ROUND " + str(roundNumber)
275
276         totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
277         for agent in agents:    
278                 totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
279                 if verbose:
280                         print "Agent: " + str(agent)
281
282         if verbose:
283                 print "Updating pretty .html files... "
284
285
286
287         for agent in agents:
288                 agentFile = open(htmlDir + agent["name"]+".html", "a")
289                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
290                 agentFile.write("<h3> Summary </h3>\n")
291                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
292                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
293                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]))+" </td> <td> "+str(len(agent["DEFEAT"]))+" </td> <td> "+str(len(agent["DRAW"]))+" </td> <td> "+str(len(agent["ILLEGAL"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
294
295                 agentFile.write("</table>\n")
296
297                 agentFile.write("<h3> Detailed </h3>\n")
298                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
299                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
300                 
301                 for index in range(0, len(agent["ALL"])):
302                         agentFile.write("<tr> <td> " + str(agent["ALL"][index][1]) + " </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
303                 agentFile.write("</table>\n")
304                 agentFile.close()       
305         
306
307 if verbose:
308         print "Finalising .html files... "
309 for agent in agents:
310         agentFile = open(htmlDir + agent["name"]+".html", "a")
311         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
312         agentFile.close()
313
314         
315 if verbose:
316         print "Done!"
317
318 endTime = time()
319 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
320 sys.exit(0)

UCC git Repository :: git.ucc.asn.au