Improved -> Broke -> Fixed basic_python AI
[progcomp2012.git] / progcomp / judge / simulator / simulate.py
1 #!/usr/bin/python -u
2
3 '''
4  simulate.py - simulation script for the 2012 UCC Programming Competition
5         NOTE: This is not the manager program for a stratego game
6         It merely calls the manager program as appropriate, and records results
7         Plays exactly ONE round, but does not overwrite previously played rounds
8         eg: run once to generate round1.results, twice to generate round2.results etc
9         Also generates total.scores based on results from every round.
10
11         Now (sortof) generates .html files to display results in a prettiful manner.
12         
13
14  author Sam Moore (matches) [SZM]
15  website http://matches.ucc.asn.au/stratego
16  email progcomp@ucc.asn.au or matches@ucc.asn.au
17  git git.ucc.asn.au/progcomp2012.git
18 '''
19
20 import os
21 import sys
22 from time import time
23
24 #Global variables/arguments
25
26 baseDirectory = "../.." #Base directory for results, logs, agents
27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
28 nRounds = 1
29
30 if len(sys.argv) >= 2:
31         nRounds = int(sys.argv[1])
32 if len(sys.argv) >= 3:
33         nGames = int(sys.argv[2])
34         if nGames % 2 != 0:
35                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
36 if len(sys.argv) >= 4:
37         baseDirectory = sys.argv[3]
38 if len(sys.argv) >= 6:
39         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
40         sys.exit(1)
41
42 resultsDirectory = baseDirectory+"/results/" #Where results will go (results are in the form of text files of agent names and scores)
43 logDirectory = baseDirectory+"/log/" #Where log files go (direct output of manager program)
44 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
45 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
46 if len(sys.argv) >= 5:
47         managerPath = sys.argv[5] 
48
49
50 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
51 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
52
53
54 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
55 verbose = True
56
57
58
59 #Check the manager program exists TODO: And is executable!
60 if os.path.exists(managerPath) == False:
61         print "Manager program at \""+managerPath+"\" doesn't exist!"
62         sys.exit(1)
63
64 #Make necessary directories
65
66 if os.path.exists(resultsDirectory) == False:
67         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
68 ''' 
69         Obselete older version doesn't work with new .html files
70 #Identify the round number by reading the results directory
71 totalRounds = len(os.listdir(resultsDirectory)) + 1
72 if totalRounds > 1:
73         totalRounds -= 1
74 '''
75
76 totalRounds = 1
77 #TODO: Fix this bit!
78
79 if os.path.exists(logDirectory) == False:
80         os.mkdir(logDirectory) #Make the log directory if it didn't exist
81
82
83 startTime = time() #Record time at which simulation starts
84
85 if verbose:
86         if nRounds > 1:
87                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
88         else:
89                 print "Simulating one round."
90         print ""
91         print "Identifying possible agents in \""+agentsDirectory+"\""
92
93 #Get all agent names from agentsDirectory
94 agentNames = os.listdir(agentsDirectory) 
95 agents = []
96 for name in agentNames:
97         if verbose:
98                 sys.stdout.write("Scan \""+name+"\"... ")
99         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
100                 if verbose:
101                         sys.stdout.write(" Invalid! (Not a directory)\n")
102                 continue
103
104         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist   
105                 if verbose:
106                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
107                 continue
108
109         agentExecutable = agentsDirectory+name+"/"+(open(agentsDirectory+name+"/info").readline().strip())
110         
111         if os.path.exists(agentExecutable) == False:
112                 if verbose:
113                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
114                 continue
115
116
117         if verbose:
118                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
119
120         #Convert array of valid names into array of dictionaries containing information about each agent
121         #I'm starting to like python...
122         agents.append({"name":name, "path":agentExecutable,  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
123
124 if len(agents) == 0:
125         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
126         sys.exit(0)
127 if verbose:
128         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
129         print ""
130
131 #Prepare the pretty .html files if they don't exist
132 if verbose:
133         print "Preparing .html results files..."
134 htmlDir = resultsDirectory + "pretty/"
135 if os.path.exists(htmlDir) == False:
136         os.mkdir(htmlDir)
137 if os.path.exists(htmlDir) == False:
138         print "Couldn't create directory \""+htmlDir+"\"."
139         sys.exit(1)
140
141 for agent in agents:
142         if os.path.exists(htmlDir+agent["name"] + ".html") == False:
143                 agentFile = open(htmlDir+agent["name"] + ".html", "w")
144                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " results</title>\n</head>\n<body>\n<h1> Results for " + agent["name"]+" </h1>\n</body>\n</html>\n")
145                 agentFile.close()
146
147         os.rename(htmlDir+agent["name"] + ".html", "tmpfile")
148         
149         oldFile = open("tmpfile", "r")
150         agentFile = open(htmlDir+agent["name"] + ".html", "w")
151         line = oldFile.readline()
152         while line != "":
153                 #if verbose:
154                 #       print "Interpreting line \"" + line.strip() + "\""
155                 if line.strip() == "</body>":
156                         break
157                 elif line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
158                         agentFile.write(line)
159                         line = oldFile.readline()
160                         
161                         values = line.split(' ')
162                         agent["totalScore"] += int(values[2].strip())
163                         agent["Wins"] += int(values[5].strip())
164                         agent["Losses"] += int(values[8].strip())
165                         agent["Draws"] += int(values[11].strip())
166                         agent["Illegal"] += int(values[14].strip())
167                         agent["Errors"] += int(values[17].strip())
168                 agentFile.write(line)
169                 line = oldFile.readline()
170
171         if verbose:
172                 print "Prepared results file \"" + htmlDir+agent["name"] + ".html\"."
173         oldFile.close()
174         agentFile.close()
175         os.remove("tmpfile")
176
177 if verbose:
178         print ""
179
180 #Do each round...
181 totalGames = nGames/2 * len(agents) * (len(agents)-1)
182 for roundNumber in range(totalRounds, totalRounds + nRounds):
183
184         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
185                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
186
187         for agent in agents:
188                 agent.update({"name":agent["name"], "path":agent["path"],  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
189
190         
191         print "Commencing ROUND " + str(roundNumber) + " combat!"
192         print "Total: " + str(totalGames) + " games to be played. This could take a while... (Estimate 60s/game)"
193
194
195
196         managerErrors = 0
197         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
198         gameNumber = 0
199         for red in agents:  #for each agent playing as red,
200                 for blue in agents: #against each other agent, playing as blue
201                         if red == blue:
202                                 continue #Exclude battles against self
203                         gameNumber += 1
204                         gameID = str(roundNumber) + "." + str(gameNumber)
205                         for i in range(1, nGames/2 + 1):
206                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
207                                 if verbose:
208                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
209                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(i)
210                                 outline = os.popen(managerPath + " -o " + logFile + " " + red["path"] + " " + blue["path"], "r").read()
211                                 results = outline.split(' ')
212                         
213                                 if len(results) != 6:
214                                         if verbose:
215                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
216                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
217                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
218                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
219                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
220                                         managerErrors += 1
221                                 else:
222
223                                         if results[1] == "RED":
224                                                 endColour = red
225                                                 otherColour = blue
226                                                 endStr = "RED"
227                                                 otherStr = "BLUE"
228                                         elif results[1] == "BLUE":
229                                                 endColour = blue
230                                                 otherColour = red
231                                                 endStr = "BLUE"
232                                                 otherStr = "RED"
233
234
235                                         if results[1] == "BOTH":
236                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
237                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
238                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
239                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
240                                                 managerErrors += 1
241                                         else:
242                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
243                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
244                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
245                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
246                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
247                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
248
249                                         
250                                         if verbose:
251                                                 sys.stdout.write(" Result \"")
252                                                 for ii in range(1, len(results)):
253                                                         sys.stdout.write(results[ii].strip())
254                                                         if ii < (len(results) - 1):
255                                                                 sys.stdout.write(" ")
256                                                 sys.stdout.write("\"\n")
257                 
258         if verbose:
259                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
260         if managerErrors != 0:
261                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
262
263         if verbose:
264                 print "" 
265         #We should now have complete score values.
266                 
267         '''
268                 Obselete, non prettified results
269         if verbose:
270                 sys.stdout.write("Creating raw results files for ROUND " + str(roundNumber) + "... ")
271
272         agents.sort(key = lambda e : e["score"], reverse=True) #Sort the agents based on score
273         
274         resultsFile = open(resultsDirectory+"round"+str(roundNumber)+".results", "w") #Create a file to store all the scores for this round
275         for agent in agents:
276                 resultsFile.write(agent["name"] + " " + str(agent["score"]) +"\n") #Write the agent names and scores into the file, in descending order
277
278         if verbose:
279                 sys.stdout.write(" Complete!\n")
280                 sys.stdout.write("Updating total scores... ");
281         
282         #Now update the total scores
283         if os.path.exists(resultsDirectory+"total.scores"):
284                 if verbose:
285                         sys.stdout.write(" Reading from \""+resultsDirectory+"total.scores\" to update scores... ")
286                 totalFile = open(resultsDirectory+"total.scores", "r") #Try to open the total.scores file
287                 for line in totalFile: #For all entries, 
288                         data = line.split(' ')
289                         for agent in agents:
290                                 if agent["name"] == data[0]:
291                                         agent["totalScore"] = int(data[1]) + agent["score"][0] #Simply increment the current score by the recorded total score of the matching file entry
292                                         break
293                 totalFile.close() #Close the file, so we can delete it
294                 os.remove(resultsDirectory+"total.scores") #Delete the file
295                 #Sort the agents again
296                 agents.sort(key = lambda e : e["totalScore"], reverse=True)
297
298         else:
299                 if verbose:
300                         sys.stdout.write(" First round - creating \""+resultsDirectory+"total.scores\"... ")
301         if verbose:
302                 sys.stdout.write(" Complete!\n")
303                 print "Finished writing results for ROUND " + str(roundNumber)
304                 print ""
305         '''
306         if verbose:     
307                 print "RESULTS FOR ROUND " + str(roundNumber)
308
309         #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
310                 for agent in agents:    
311                 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
312                 #if verbose:
313                                 print "Agent: " + str(agent)
314         
315
316         if verbose:
317                 print "Updating pretty .html files... "
318
319         for agent in agents:
320                 agentFile = open(htmlDir + agent["name"]+".html", "a")
321                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
322                 agentFile.write("<h3> Round Overview </h3>\n")
323                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
324                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
325                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
326
327                 agentFile.write("</table>\n")
328
329                 agentFile.write("<h3> Detailed </h3>\n")
330                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
331                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
332                 
333                 for index in range(0, len(agent["ALL"])):
334                         agentFile.write("<tr> <td> " + str(agent["ALL"][index][1]) + " </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
335                 agentFile.write("</table>\n")
336                 
337                 agent["totalScore"] += agent["score"][0]
338                 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
339                 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
340                 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
341                 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
342                 agent["Errors"] += len(agent["INTERNAL_ERROR"])
343
344                 agentFile.write("<h3> Accumulated Results </h3>\n")
345                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
346                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
347                 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
348
349                 agentFile.write("</table>\n")
350                 agentFile.close()       
351
352         #Update round file
353         roundFile = open(htmlDir + "round"+str(roundNumber)+".html", "w")
354         roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
355         roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
356         roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
357         roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
358         agents.sort(key = lambda e : e["score"][0], reverse=True)
359         for agent in agents:
360                 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
361         roundFile.write("</table>\n</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
362         roundFile.close()
363
364
365         
366         
367
368 if verbose:
369         print "Finalising .html files... "
370 for agent in agents:
371         agentFile = open(htmlDir + agent["name"]+".html", "a")
372
373         #Write the "total" statistics
374
375         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
376         agentFile.close()
377
378         if os.path.exists(htmlDir + "total.html") == True:
379                 os.remove(htmlDir + "total.html") #Delete the file
380
381 totalFile = open(htmlDir + "total.html", "w")
382 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
383 totalFile.write("<h1> Total Overview </h1>\n")
384 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
385 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
386 agents.sort(key = lambda e : e["totalScore"], reverse=True)
387 for agent in agents:
388         totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
389 totalFile.write("</table>\n</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
390 totalFile.close()
391
392         
393 if verbose:
394         print "Done!"
395
396 endTime = time()
397 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
398 sys.exit(0)

UCC git Repository :: git.ucc.asn.au