progcomp/judge/simulator/simulate.py

   1 #!/usr/bin/python -u
   2
   3 '''
   4  simulate.py - simulation script for the 2012 UCC Programming Competition
   5         NOTE: This is not the manager program for a stratego game
   6         It merely calls the manager program as appropriate, and records results
   7         Plays exactly ONE round, but does not overwrite previously played rounds
   8         eg: run once to generate round1.results, twice to generate round2.results etc
   9         Also generates total.scores based on results from every round.
  10
  11         Now (sortof) generates .html files to display results in a prettiful manner.
  12
  13
  14  author Sam Moore (matches) [SZM]
  15  website http://matches.ucc.asn.au/stratego
  16  email [email protected] or [email protected]
  17  git git.ucc.asn.au/progcomp2012.git
  18 '''
  19
  20 import os
  21 import sys
  22 from time import time
  23
  24 #Global variables/arguments
  25
  26 baseDirectory = "../.." #Base directory for results, logs, agents
  27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
  28 nRounds = 1
  29
  30 if len(sys.argv) >= 2:
  31         nRounds = int(sys.argv[1])
  32 if len(sys.argv) >= 3:
  33         nGames = int(sys.argv[2])
  34         if nGames % 2 != 0:
  35                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
  36 if len(sys.argv) >= 4:
  37         baseDirectory = sys.argv[3]
  38 if len(sys.argv) >= 6:
  39         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
  40         sys.exit(1)
  41
  42 resultsDirectory = baseDirectory+"/results/" #Where results will go (results are in the form of text files of agent names and scores)
  43 logDirectory = baseDirectory+"/log/" #Where log files go (direct output of manager program)
  44 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
  45 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
  46 if len(sys.argv) >= 5:
  47         managerPath = sys.argv[5]
  48
  49
  50 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
  51 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
  52
  53
  54 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
  55 verbose = True
  56
  57
  58
  59 #Check the manager program exists TODO: And is executable!
  60 if os.path.exists(managerPath) == False:
  61         print "Manager program at \""+managerPath+"\" doesn't exist!"
  62         sys.exit(1)
  63
  64 #Make necessary directories
  65
  66 if os.path.exists(resultsDirectory) == False:
  67         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
  68 #Identify the round number by reading the results directory
  69 totalRounds = len(os.listdir(resultsDirectory)) + 1
  70 if totalRounds > 1:
  71         totalRounds -= 1
  72
  73 if os.path.exists(logDirectory) == False:
  74         os.mkdir(logDirectory) #Make the log directory if it didn't exist
  75
  76
  77 startTime = time() #Record time at which simulation starts
  78
  79 if verbose:
  80         if nRounds > 1:
  81                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
  82         else:
  83                 print "Simulating one round."
  84         print ""
  85         print "Identifying possible agents in \""+agentsDirectory+"\""
  86
  87 #Get all agent names from agentsDirectory
  88 agentNames = os.listdir(agentsDirectory)
  89 agents = []
  90 for name in agentNames:
  91         if verbose:
  92                 sys.stdout.write("Scan \""+name+"\"... ")
  93         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
  94                 if verbose:
  95                         sys.stdout.write(" Invalid! (Not a directory)\n")
  96                 continue
  97
  98         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist
  99                 if verbose:
 100                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
 101                 continue
 102
 103         agentExecutable = agentsDirectory+name+"/"+(open(agentsDirectory+name+"/info").readline().strip())
 104
 105         if os.path.exists(agentExecutable) == False:
 106                 if verbose:
 107                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
 108                 continue
 109
 110
 111         if verbose:
 112                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
 113
 114         #Convert array of valid names into array of dictionaries containing information about each agent
 115         #I'm starting to like python...
 116         agents.append({"name":name, "path":agentExecutable,  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
 117
 118 if len(agents) == 0:
 119         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
 120         sys.exit(0)
 121 if verbose:
 122         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
 123         print ""
 124
 125 #Prepare the pretty .html files if they don't exist
 126 if verbose:
 127         print "Preparing .html results files..."
 128 htmlDir = resultsDirectory + "pretty/"
 129 if os.path.exists(htmlDir) == False:
 130         os.mkdir(htmlDir)
 131 if os.path.exists(htmlDir) == False:
 132         print "Couldn't create directory \""+htmlDir+"\"."
 133         sys.exit(1)
 134
 135 for agent in agents:
 136         if os.path.exists(htmlDir+agent["name"] + ".html") == False:
 137                 agentFile = open(htmlDir+agent["name"] + ".html", "w")
 138                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " results</title>\n</head>\n<body>\n<h1> Results for " + agent["name"]+" </h1>\n</body>\n</html>\n")
 139                 agentFile.close()
 140
 141         os.rename(htmlDir+agent["name"] + ".html", "tmpfile")
 142
 143         oldFile = open("tmpfile", "r")
 144         agentFile = open(htmlDir+agent["name"] + ".html", "w")
 145         line = oldFile.readline()
 146         while line != "":
 147                 #if verbose:
 148                 #       print "Interpreting line \"" + line.strip() + "\""
 149                 if line.strip() == "</body>":
 150                         break
 151                 elif line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
 152                         agentFile.write(line)
 153                         line = oldFile.readline()
 154
 155                         values = line.split(' ')
 156                         agent["totalScore"] += int(values[2].strip())
 157                         agent["Wins"] += int(values[5].strip())
 158                         agent["Losses"] += int(values[8].strip())
 159                         agent["Draws"] += int(values[11].strip())
 160                         agent["Illegal"] += int(values[14].strip())
 161                         agent["Errors"] += int(values[17].strip())
 162                 agentFile.write(line)
 163                 line = oldFile.readline()
 164
 165         if verbose:
 166                 print "Prepared results file \"" + htmlDir+agent["name"] + ".html\"."
 167         oldFile.close()
 168         agentFile.close()
 169         os.remove("tmpfile")
 170
 171 if verbose:
 172         print ""
 173
 174 #Do each round...
 175 totalGames = nGames/2 * len(agents) * (len(agents)-1)
 176 for roundNumber in range(totalRounds, totalRounds + nRounds):
 177
 178         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
 179                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
 180
 181         for agent in agents:
 182                 agent.update({"name":agent["name"], "path":agent["path"],  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
 183
 184
 185         print "Commencing ROUND " + str(roundNumber) + " combat!"
 186         print "Total: " + str(totalGames) + " games to be played. This could take a while... (Estimate 60s/game)"
 187
 188
 189
 190         managerErrors = 0
 191         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
 192         gameNumber = 0
 193         for red in agents:  #for each agent playing as red,
 194                 for blue in agents: #against each other agent, playing as blue
 195                         if red == blue:
 196                                 continue #Exclude battles against self
 197                         gameNumber += 1
 198                         gameID = str(roundNumber) + "." + str(gameNumber)
 199                         for i in range(1, nGames/2 + 1):
 200                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
 201                                 if verbose:
 202                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
 203                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(i)
 204                                 outline = os.popen(managerPath + " -o " + logFile + " " + red["path"] + " " + blue["path"], "r").read()
 205                                 results = outline.split(' ')
 206
 207                                 if len(results) != 6:
 208                                         if verbose:
 209                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
 210                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
 211                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
 212                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
 213                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
 214                                         managerErrors += 1
 215                                 else:
 216
 217                                         if results[1] == "RED":
 218                                                 endColour = red
 219                                                 otherColour = blue
 220                                                 endStr = "RED"
 221                                                 otherStr = "BLUE"
 222                                         elif results[1] == "BLUE":
 223                                                 endColour = blue
 224                                                 otherColour = red
 225                                                 endStr = "BLUE"
 226                                                 otherStr = "RED"
 227
 228
 229                                         if results[1] == "BOTH":
 230                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
 231                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
 232                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
 233                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
 234                                                 managerErrors += 1
 235                                         else:
 236                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
 237                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
 238                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
 239                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
 240                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
 241                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
 242
 243
 244                                         if verbose:
 245                                                 sys.stdout.write(" Result \"")
 246                                                 for ii in range(1, len(results)):
 247                                                         sys.stdout.write(results[ii].strip())
 248                                                         if ii < (len(results) - 1):
 249                                                                 sys.stdout.write(" ")
 250                                                 sys.stdout.write("\"\n")
 251
 252         if verbose:
 253                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
 254         if managerErrors != 0:
 255                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
 256
 257         if verbose:
 258                 print ""
 259         #We should now have complete score values.
 260
 261         '''
 262                 Obselete, non prettified results
 263         if verbose:
 264                 sys.stdout.write("Creating raw results files for ROUND " + str(roundNumber) + "... ")
 265
 266         agents.sort(key = lambda e : e["score"], reverse=True) #Sort the agents based on score
 267
 268         resultsFile = open(resultsDirectory+"round"+str(roundNumber)+".results", "w") #Create a file to store all the scores for this round
 269         for agent in agents:
 270                 resultsFile.write(agent["name"] + " " + str(agent["score"]) +"\n") #Write the agent names and scores into the file, in descending order
 271
 272         if verbose:
 273                 sys.stdout.write(" Complete!\n")
 274                 sys.stdout.write("Updating total scores... ");
 275
 276         #Now update the total scores
 277         if os.path.exists(resultsDirectory+"total.scores"):
 278                 if verbose:
 279                         sys.stdout.write(" Reading from \""+resultsDirectory+"total.scores\" to update scores... ")
 280                 totalFile = open(resultsDirectory+"total.scores", "r") #Try to open the total.scores file
 281                 for line in totalFile: #For all entries,
 282                         data = line.split(' ')
 283                         for agent in agents:
 284                                 if agent["name"] == data[0]:
 285                                         agent["totalScore"] = int(data[1]) + agent["score"][0] #Simply increment the current score by the recorded total score of the matching file entry
 286                                         break
 287                 totalFile.close() #Close the file, so we can delete it
 288                 os.remove(resultsDirectory+"total.scores") #Delete the file
 289                 #Sort the agents again
 290                 agents.sort(key = lambda e : e["totalScore"], reverse=True)
 291
 292         else:
 293                 if verbose:
 294                         sys.stdout.write(" First round - creating \""+resultsDirectory+"total.scores\"... ")
 295         if verbose:
 296                 sys.stdout.write(" Complete!\n")
 297                 print "Finished writing results for ROUND " + str(roundNumber)
 298                 print ""
 299         '''
 300         if verbose:
 301                 print "RESULTS FOR ROUND " + str(roundNumber)
 302
 303         #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
 304                 for agent in agents:
 305                 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
 306                 #if verbose:
 307                                 print "Agent: " + str(agent)
 308
 309
 310         if verbose:
 311                 print "Updating pretty .html files... "
 312
 313         for agent in agents:
 314                 agentFile = open(htmlDir + agent["name"]+".html", "a")
 315                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
 316                 agentFile.write("<h3> Round Overview </h3>\n")
 317                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 318                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
 319                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
 320
 321                 agentFile.write("</table>\n")
 322
 323                 agentFile.write("<h3> Detailed </h3>\n")
 324                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 325                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
 326
 327                 for index in range(0, len(agent["ALL"])):
 328                         agentFile.write("<tr> <td> " + str(agent["ALL"][index][1]) + " </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
 329                 agentFile.write("</table>\n")
 330
 331                 agent["totalScore"] += agent["score"][0]
 332                 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
 333                 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
 334                 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
 335                 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
 336                 agent["Errors"] += len(agent["INTERNAL_ERROR"])
 337
 338                 agentFile.write("<h3> Accumulated Results </h3>\n")
 339                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 340                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
 341                 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
 342
 343                 agentFile.write("</table>\n")
 344                 agentFile.close()
 345
 346
 347
 348
 349 if verbose:
 350         print "Finalising .html files... "
 351 for agent in agents:
 352         agentFile = open(htmlDir + agent["name"]+".html", "a")
 353
 354         #Write the "total" statistics
 355
 356         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
 357         agentFile.close()
 358
 359
 360 if verbose:
 361         print "Done!"
 362
 363 endTime = time()
 364 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
 365 sys.exit(0)