aead755beaafc63361e2b9a6375d6791ee7ca236
[progcomp2012.git] / judge / simulator / simulate.py
1 #!/usr/bin/python -u
2
3 '''
4  simulate.py - simulation script for the 2012 UCC Programming Competition
5         NOTE: This is not the manager program for a stratego game
6         It merely calls the manager program as appropriate, and records results
7         Plays exactly ONE round, but does not overwrite previously played rounds
8         eg: run once to generate round1.results, twice to generate round2.results etc
9         Also generates total.scores based on results from every round.
10
11         Now (sortof) generates .html files to display results in a prettiful manner.
12         
13
14  author Sam Moore (matches) [SZM]
15  website http://matches.ucc.asn.au/stratego
16  email progcomp@ucc.asn.au or matches@ucc.asn.au
17  git git.ucc.asn.au/progcomp2012.git
18 '''
19
20 import os
21 import sys
22 from time import time
23
24 #Global variables/arguments
25
26 baseDirectory = "../.." #Base directory for results, logs, agents
27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
28 nRounds = 1
29
30 if len(sys.argv) >= 2:
31         nRounds = int(sys.argv[1])
32 if len(sys.argv) >= 3:
33         nGames = int(sys.argv[2])
34         if nGames % 2 != 0:
35                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
36 if len(sys.argv) >= 4:
37         baseDirectory = sys.argv[3]
38 if len(sys.argv) >= 6:
39         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
40         sys.exit(1)
41
42 resultsDirectory = baseDirectory+"/web/results/" #Where results will go (results are in the form of text files of agent names and scores)
43 logDirectory = baseDirectory+"/web/log/" #Where log files go (direct output of manager program)
44 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
45 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
46 if len(sys.argv) >= 5:
47         managerPath = sys.argv[5] 
48
49
50 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
51 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
52
53
54 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
55 verbose = True
56
57
58
59 #Check the manager program exists TODO: And is executable!
60 if os.path.exists(managerPath) == False:
61         print "Manager program at \""+managerPath+"\" doesn't exist!"
62         sys.exit(1)
63
64 #Make necessary directories
65
66 if os.path.exists(resultsDirectory) == False:
67         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
68
69         
70 #Identify the round number by reading from the "info" file in the results directory, if it doesn't exist then start at round 1.
71 if os.path.exists(resultsDirectory+"info") == False:
72         totalRounds = 1
73 else:
74         info = open(resultsDirectory+"info", "r")
75         totalRounds = int(info.readline().strip())
76         info.close()
77         os.remove(resultsDirectory+"info")
78
79 info = open(resultsDirectory+"info", "w")
80 info.write(str(totalRounds + nRounds) + "\n")
81 info.close()
82         
83
84
85 if os.path.exists(logDirectory) == False:
86         os.mkdir(logDirectory) #Make the log directory if it didn't exist
87
88
89 startTime = time() #Record time at which simulation starts
90
91 if verbose:
92         if nRounds > 1:
93                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
94         else:
95                 print "Simulating one round."
96         print ""
97         print "Identifying possible agents in \""+agentsDirectory+"\""
98
99 #Get all agent names from agentsDirectory
100 agentNames = os.listdir(agentsDirectory) 
101 agents = []
102 for name in agentNames:
103         if verbose:
104                 sys.stdout.write("Scan \""+name+"\"... ")
105         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
106                 if verbose:
107                         sys.stdout.write(" Invalid! (Not a directory)\n")
108                 continue
109
110         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist   
111                 if verbose:
112                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
113                 continue
114
115         infoFile = open(agentsDirectory+name+"/info", "r")
116         agentExecutable = agentsDirectory+name+"/"+(infoFile.readline().strip())
117         author = infoFile.readline().strip()
118         language = infoFile.readline().strip()
119         description = ""
120         while True:
121                 line = infoFile.readline()
122                 if len(line) > 0:
123                         description += line
124                 else:
125                         break
126         infoFile.close()
127         
128         if os.path.exists(agentExecutable) == False:
129                 if verbose:
130                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
131                 continue
132
133
134         if verbose:
135                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
136
137         #Convert array of valid names into array of dictionaries containing information about each agent
138         #I'm starting to like python...
139         agents.append({"name":name, "path":agentExecutable, "author":author, "language":language, "description":description, "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
140
141 if len(agents) == 0:
142         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
143         sys.exit(0)
144 if verbose:
145         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
146         print ""
147
148 #Prepare the pretty .html files if they don't exist
149 if verbose:
150         print "Preparing .html results files..."
151
152
153 for agent in agents:
154         if os.path.exists(resultsDirectory+agent["name"] + ".html") == False:
155                 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
156                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " overview</title>\n</head>\n<body>\n<h1> Overview for " + agent["name"]+" </h1>\n")
157                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
158                 agentFile.write("<tr> <th> Name </th> <th> Author </th> <th> Language </th> </tr>\n")
159                 agentFile.write("<tr> <td> "+agent["name"]+" </td> <td> "+agent["author"]+" </td> <td> "+agent["language"]+" </td> </tr>\n")
160                 agentFile.write("</table>\n");
161
162                 agentFile.write("<p> <b>Description</b> </p>\n")
163                 agentFile.write("<p> " + agent["description"] + " </p>\n")
164                 agentFile.close()
165
166         os.rename(resultsDirectory+agent["name"] + ".html", "tmpfile")
167         
168         oldFile = open("tmpfile", "r")
169         agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
170         line = oldFile.readline()
171         while line != "":
172                 #if verbose:
173                 #       print "Interpreting line \"" + line.strip() + "\""
174                 if line.strip() == "</body>":
175                         break
176                 elif line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
177                         agentFile.write(line)
178                         line = oldFile.readline()
179                         
180                         values = line.split(' ')
181                         agent["totalScore"] += int(values[2].strip())
182                         agent["Wins"] += int(values[5].strip())
183                         agent["Losses"] += int(values[8].strip())
184                         agent["Draws"] += int(values[11].strip())
185                         agent["Illegal"] += int(values[14].strip())
186                         agent["Errors"] += int(values[17].strip())
187                 agentFile.write(line)
188                 line = oldFile.readline()
189
190         if verbose:
191                 print "Prepared results file \"" + resultsDirectory+agent["name"] + ".html\"."
192         oldFile.close()
193         agentFile.close()
194         os.remove("tmpfile")
195
196 if verbose:
197         print ""
198
199 #Do each round...
200 totalGames = nGames/2 * len(agents) * (len(agents)-1)
201 for roundNumber in range(totalRounds, totalRounds + nRounds):
202
203         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
204                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
205
206         for agent in agents:
207                 agent.update({"name":agent["name"], "path":agent["path"],  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
208
209         
210         print "Commencing ROUND " + str(roundNumber) + " combat!"
211         print "Total: " + str(totalGames) + " games to be played. This could take a while... (Estimate 60s/game)"
212
213
214
215         managerErrors = 0
216         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
217         gameNumber = 0
218         for red in agents:  #for each agent playing as red,
219                 for blue in agents: #against each other agent, playing as blue
220                         if red == blue:
221                                 continue #Exclude battles against self
222                         gameNumber += 1
223                         gameID = str(roundNumber) + "." + str(gameNumber)
224                         for i in range(1, nGames/2 + 1):
225                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
226                                 if verbose:
227                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
228                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(gameID)
229                                 outline = os.popen(managerPath + " -o " + logFile + " " + red["path"] + " " + blue["path"], "r").read()
230                                 results = outline.split(' ')
231                         
232                                 if len(results) != 6:
233                                         if verbose:
234                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
235                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
236                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
237                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
238                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
239                                         managerErrors += 1
240                                 else:
241
242                                         if results[1] == "RED":
243                                                 endColour = red
244                                                 otherColour = blue
245                                                 endStr = "RED"
246                                                 otherStr = "BLUE"
247                                         elif results[1] == "BLUE":
248                                                 endColour = blue
249                                                 otherColour = red
250                                                 endStr = "BLUE"
251                                                 otherStr = "RED"
252
253
254                                         if results[1] == "BOTH":
255                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
256                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
257                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
258                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
259                                                 managerErrors += 1
260                                         else:
261                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
262                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
263                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
264                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
265                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
266                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
267
268                                         
269                                         if verbose:
270                                                 sys.stdout.write(" Result \"")
271                                                 for ii in range(1, len(results)):
272                                                         sys.stdout.write(results[ii].strip())
273                                                         if ii < (len(results) - 1):
274                                                                 sys.stdout.write(" ")
275                                                 sys.stdout.write("\"\n")
276                 
277         if verbose:
278                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
279         if managerErrors != 0:
280                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
281
282         if verbose:
283                 print "" 
284         #We should now have complete score values.
285                 
286         '''
287                 Obselete, non prettified results
288         if verbose:
289                 sys.stdout.write("Creating raw results files for ROUND " + str(roundNumber) + "... ")
290
291         agents.sort(key = lambda e : e["score"], reverse=True) #Sort the agents based on score
292         
293         resultsFile = open(resultsDirectory+"round"+str(roundNumber)+".results", "w") #Create a file to store all the scores for this round
294         for agent in agents:
295                 resultsFile.write(agent["name"] + " " + str(agent["score"]) +"\n") #Write the agent names and scores into the file, in descending order
296
297         if verbose:
298                 sys.stdout.write(" Complete!\n")
299                 sys.stdout.write("Updating total scores... ");
300         
301         #Now update the total scores
302         if os.path.exists(resultsDirectory+"total.scores"):
303                 if verbose:
304                         sys.stdout.write(" Reading from \""+resultsDirectory+"total.scores\" to update scores... ")
305                 totalFile = open(resultsDirectory+"total.scores", "r") #Try to open the total.scores file
306                 for line in totalFile: #For all entries, 
307                         data = line.split(' ')
308                         for agent in agents:
309                                 if agent["name"] == data[0]:
310                                         agent["totalScore"] = int(data[1]) + agent["score"][0] #Simply increment the current score by the recorded total score of the matching file entry
311                                         break
312                 totalFile.close() #Close the file, so we can delete it
313                 os.remove(resultsDirectory+"total.scores") #Delete the file
314                 #Sort the agents again
315                 agents.sort(key = lambda e : e["totalScore"], reverse=True)
316
317         else:
318                 if verbose:
319                         sys.stdout.write(" First round - creating \""+resultsDirectory+"total.scores\"... ")
320         if verbose:
321                 sys.stdout.write(" Complete!\n")
322                 print "Finished writing results for ROUND " + str(roundNumber)
323                 print ""
324         '''
325         if verbose:     
326                 print "RESULTS FOR ROUND " + str(roundNumber)
327
328         #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
329                 for agent in agents:    
330                 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
331                 #if verbose:
332                                 print "Agent: " + str(agent)
333         
334
335         if verbose:
336                 print "Updating pretty .html files... "
337
338         for agent in agents:
339                 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
340                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
341                 agentFile.write("<h3> Round Overview </h3>\n")
342                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
343                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
344                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
345
346                 agentFile.write("</table>\n")
347                 agentFile.write("<p> <a href=round"+str(roundNumber)+".html>Round "+str(roundNumber) + " Scoreboard</a></p>\n")
348
349                 agentFile.write("<h3> Detailed </h3>\n")
350                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
351                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
352                 
353                 
354
355                 for index in range(0, len(agent["ALL"])):
356                         if agent["ALL"][index][4] == "RED":
357                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+agent["name"]+".vs."+agent["ALL"][index][0]+"."+str(agent["ALL"][index][1])
358                         else:
359                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+agent["ALL"][index][0]+".vs."+agent["name"]+"."+str(agent["ALL"][index][1])
360                         agentFile.write("<tr> <td> <a href="+logFile+">" + str(agent["ALL"][index][1]) + " </a> </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
361                 agentFile.write("</table>\n")
362                 
363                 agent["totalScore"] += agent["score"][0]
364                 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
365                 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
366                 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
367                 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
368                 agent["Errors"] += len(agent["INTERNAL_ERROR"])
369
370                 agentFile.write("<h3> Accumulated Results </h3>\n")
371                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
372                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
373                 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
374
375                 agentFile.write("</table>\n")
376
377
378                 agentFile.close()       
379
380         #Update round file
381         roundFile = open(resultsDirectory + "round"+str(roundNumber)+".html", "w")
382         roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
383         roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
384         roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
385         roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
386         agents.sort(key = lambda e : e["score"][0], reverse=True)
387         for agent in agents:
388                 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
389         roundFile.write("</table>\n")
390         roundFile.write("<p> <a href=total.html>Current Scoreboard</a></p>\n")
391         roundFile.write("</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
392         roundFile.close()
393
394
395         
396         
397
398 if verbose:
399         print "Finalising .html files... "
400 for agent in agents:
401         agentFile = open(resultsDirectory + agent["name"]+".html", "a")
402
403         #Write the "total" statistics
404
405         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
406         agentFile.close()
407
408         if os.path.exists(resultsDirectory + "total.html") == True:
409                 os.remove(resultsDirectory + "total.html") #Delete the file
410
411 totalFile = open(resultsDirectory + "total.html", "w")
412 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
413 totalFile.write("<h1> Total Overview </h1>\n")
414 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
415 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
416 agents.sort(key = lambda e : e["totalScore"], reverse=True)
417 for agent in agents:
418         totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
419 totalFile.write("</table>\n")
420
421 totalFile.write("<h2> Round Summaries </h2>\n")
422 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
423 for i in range(1, totalRounds+1):
424         totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
425 totalFile.write("</table>\n")
426
427 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
428 totalFile.close()
429
430         
431 if verbose:
432         print "Done!"
433
434 endTime = time()
435 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
436 sys.exit(0)

UCC git Repository :: git.ucc.asn.au