progcomp/judge/simulator/simulate.py

   1 #!/usr/bin/python -u
   2
   3 '''
   4  simulate.py - simulation script for the 2012 UCC Programming Competition
   5         NOTE: This is not the manager program for a stratego game
   6         It merely calls the manager program as appropriate, and records results
   7         Plays exactly ONE round, but does not overwrite previously played rounds
   8         eg: run once to generate round1.results, twice to generate round2.results etc
   9         Also generates total.scores based on results from every round.
  10
  11         Now (sortof) generates .html files to display results in a prettiful manner.
  12
  13
  14  author Sam Moore (matches) [SZM]
  15  website http://matches.ucc.asn.au/stratego
  16  email [email protected] or [email protected]
  17  git git.ucc.asn.au/progcomp2012.git
  18 '''
  19
  20 import os
  21 import sys
  22 from time import time
  23
  24 #Global variables/arguments
  25
  26 baseDirectory = "../.." #Base directory for results, logs, agents
  27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
  28 nRounds = 1
  29
  30 if len(sys.argv) >= 2:
  31         nRounds = int(sys.argv[1])
  32 if len(sys.argv) >= 3:
  33         nGames = int(sys.argv[2])
  34         if nGames % 2 != 0:
  35                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
  36 if len(sys.argv) >= 4:
  37         baseDirectory = sys.argv[3]
  38 if len(sys.argv) >= 6:
  39         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
  40         sys.exit(1)
  41
  42 resultsDirectory = baseDirectory+"/results/" #Where results will go (results are in the form of text files of agent names and scores)
  43 logDirectory = baseDirectory+"/log/" #Where log files go (direct output of manager program)
  44 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
  45 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
  46 if len(sys.argv) >= 5:
  47         managerPath = sys.argv[5]
  48
  49
  50 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
  51 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
  52
  53
  54 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
  55 verbose = True
  56
  57
  58
  59 #Check the manager program exists TODO: And is executable!
  60 if os.path.exists(managerPath) == False:
  61         print "Manager program at \""+managerPath+"\" doesn't exist!"
  62         sys.exit(1)
  63
  64 #Make necessary directories
  65
  66 if os.path.exists(resultsDirectory) == False:
  67         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
  68 '''
  69         Obselete older version doesn't work with new .html files
  70 #Identify the round number by reading the results directory
  71 totalRounds = len(os.listdir(resultsDirectory)) + 1
  72 if totalRounds > 1:
  73         totalRounds -= 1
  74 '''
  75
  76 totalRounds = 1
  77 #TODO: Fix this bit!
  78
  79 if os.path.exists(logDirectory) == False:
  80         os.mkdir(logDirectory) #Make the log directory if it didn't exist
  81
  82
  83 startTime = time() #Record time at which simulation starts
  84
  85 if verbose:
  86         if nRounds > 1:
  87                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
  88         else:
  89                 print "Simulating one round."
  90         print ""
  91         print "Identifying possible agents in \""+agentsDirectory+"\""
  92
  93 #Get all agent names from agentsDirectory
  94 agentNames = os.listdir(agentsDirectory)
  95 agents = []
  96 for name in agentNames:
  97         if verbose:
  98                 sys.stdout.write("Scan \""+name+"\"... ")
  99         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
 100                 if verbose:
 101                         sys.stdout.write(" Invalid! (Not a directory)\n")
 102                 continue
 103
 104         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist
 105                 if verbose:
 106                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
 107                 continue
 108
 109         agentExecutable = agentsDirectory+name+"/"+(open(agentsDirectory+name+"/info").readline().strip())
 110
 111         if os.path.exists(agentExecutable) == False:
 112                 if verbose:
 113                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
 114                 continue
 115
 116
 117         if verbose:
 118                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
 119
 120         #Convert array of valid names into array of dictionaries containing information about each agent
 121         #I'm starting to like python...
 122         agents.append({"name":name, "path":agentExecutable,  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
 123
 124 if len(agents) == 0:
 125         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
 126         sys.exit(0)
 127 if verbose:
 128         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
 129         print ""
 130
 131 #Prepare the pretty .html files if they don't exist
 132 if verbose:
 133         print "Preparing .html results files..."
 134 htmlDir = resultsDirectory + "pretty/"
 135 if os.path.exists(htmlDir) == False:
 136         os.mkdir(htmlDir)
 137 if os.path.exists(htmlDir) == False:
 138         print "Couldn't create directory \""+htmlDir+"\"."
 139         sys.exit(1)
 140
 141 for agent in agents:
 142         if os.path.exists(htmlDir+agent["name"] + ".html") == False:
 143                 agentFile = open(htmlDir+agent["name"] + ".html", "w")
 144                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " results</title>\n</head>\n<body>\n<h1> Results for " + agent["name"]+" </h1>\n</body>\n</html>\n")
 145                 agentFile.close()
 146
 147         os.rename(htmlDir+agent["name"] + ".html", "tmpfile")
 148
 149         oldFile = open("tmpfile", "r")
 150         agentFile = open(htmlDir+agent["name"] + ".html", "w")
 151         line = oldFile.readline()
 152         while line != "":
 153                 #if verbose:
 154                 #       print "Interpreting line \"" + line.strip() + "\""
 155                 if line.strip() == "</body>":
 156                         break
 157                 elif line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
 158                         agentFile.write(line)
 159                         line = oldFile.readline()
 160
 161                         values = line.split(' ')
 162                         agent["totalScore"] += int(values[2].strip())
 163                         agent["Wins"] += int(values[5].strip())
 164                         agent["Losses"] += int(values[8].strip())
 165                         agent["Draws"] += int(values[11].strip())
 166                         agent["Illegal"] += int(values[14].strip())
 167                         agent["Errors"] += int(values[17].strip())
 168                 agentFile.write(line)
 169                 line = oldFile.readline()
 170
 171         if verbose:
 172                 print "Prepared results file \"" + htmlDir+agent["name"] + ".html\"."
 173         oldFile.close()
 174         agentFile.close()
 175         os.remove("tmpfile")
 176
 177 if verbose:
 178         print ""
 179
 180 #Do each round...
 181 totalGames = nGames/2 * len(agents) * (len(agents)-1)
 182 for roundNumber in range(totalRounds, totalRounds + nRounds):
 183
 184         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
 185                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
 186
 187         for agent in agents:
 188                 agent.update({"name":agent["name"], "path":agent["path"],  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
 189
 190
 191         print "Commencing ROUND " + str(roundNumber) + " combat!"
 192         print "Total: " + str(totalGames) + " games to be played. This could take a while... (Estimate 60s/game)"
 193
 194
 195
 196         managerErrors = 0
 197         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
 198         gameNumber = 0
 199         for red in agents:  #for each agent playing as red,
 200                 for blue in agents: #against each other agent, playing as blue
 201                         if red == blue:
 202                                 continue #Exclude battles against self
 203                         gameNumber += 1
 204                         gameID = str(roundNumber) + "." + str(gameNumber)
 205                         for i in range(1, nGames/2 + 1):
 206                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
 207                                 if verbose:
 208                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
 209                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(i)
 210                                 outline = os.popen(managerPath + " -o " + logFile + " " + red["path"] + " " + blue["path"], "r").read()
 211                                 results = outline.split(' ')
 212
 213                                 if len(results) != 6:
 214                                         if verbose:
 215                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
 216                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
 217                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
 218                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
 219                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
 220                                         managerErrors += 1
 221                                 else:
 222
 223                                         if results[1] == "RED":
 224                                                 endColour = red
 225                                                 otherColour = blue
 226                                                 endStr = "RED"
 227                                                 otherStr = "BLUE"
 228                                         elif results[1] == "BLUE":
 229                                                 endColour = blue
 230                                                 otherColour = red
 231                                                 endStr = "BLUE"
 232                                                 otherStr = "RED"
 233
 234
 235                                         if results[1] == "BOTH":
 236                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
 237                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
 238                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
 239                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
 240                                                 managerErrors += 1
 241                                         else:
 242                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
 243                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
 244                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
 245                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
 246                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
 247                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
 248
 249
 250                                         if verbose:
 251                                                 sys.stdout.write(" Result \"")
 252                                                 for ii in range(1, len(results)):
 253                                                         sys.stdout.write(results[ii].strip())
 254                                                         if ii < (len(results) - 1):
 255                                                                 sys.stdout.write(" ")
 256                                                 sys.stdout.write("\"\n")
 257
 258         if verbose:
 259                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
 260         if managerErrors != 0:
 261                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
 262
 263         if verbose:
 264                 print ""
 265         #We should now have complete score values.
 266
 267         '''
 268                 Obselete, non prettified results
 269         if verbose:
 270                 sys.stdout.write("Creating raw results files for ROUND " + str(roundNumber) + "... ")
 271
 272         agents.sort(key = lambda e : e["score"], reverse=True) #Sort the agents based on score
 273
 274         resultsFile = open(resultsDirectory+"round"+str(roundNumber)+".results", "w") #Create a file to store all the scores for this round
 275         for agent in agents:
 276                 resultsFile.write(agent["name"] + " " + str(agent["score"]) +"\n") #Write the agent names and scores into the file, in descending order
 277
 278         if verbose:
 279                 sys.stdout.write(" Complete!\n")
 280                 sys.stdout.write("Updating total scores... ");
 281
 282         #Now update the total scores
 283         if os.path.exists(resultsDirectory+"total.scores"):
 284                 if verbose:
 285                         sys.stdout.write(" Reading from \""+resultsDirectory+"total.scores\" to update scores... ")
 286                 totalFile = open(resultsDirectory+"total.scores", "r") #Try to open the total.scores file
 287                 for line in totalFile: #For all entries,
 288                         data = line.split(' ')
 289                         for agent in agents:
 290                                 if agent["name"] == data[0]:
 291                                         agent["totalScore"] = int(data[1]) + agent["score"][0] #Simply increment the current score by the recorded total score of the matching file entry
 292                                         break
 293                 totalFile.close() #Close the file, so we can delete it
 294                 os.remove(resultsDirectory+"total.scores") #Delete the file
 295                 #Sort the agents again
 296                 agents.sort(key = lambda e : e["totalScore"], reverse=True)
 297
 298         else:
 299                 if verbose:
 300                         sys.stdout.write(" First round - creating \""+resultsDirectory+"total.scores\"... ")
 301         if verbose:
 302                 sys.stdout.write(" Complete!\n")
 303                 print "Finished writing results for ROUND " + str(roundNumber)
 304                 print ""
 305         '''
 306         if verbose:
 307                 print "RESULTS FOR ROUND " + str(roundNumber)
 308
 309         #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
 310                 for agent in agents:
 311                 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
 312                 #if verbose:
 313                                 print "Agent: " + str(agent)
 314
 315
 316         if verbose:
 317                 print "Updating pretty .html files... "
 318
 319         for agent in agents:
 320                 agentFile = open(htmlDir + agent["name"]+".html", "a")
 321                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
 322                 agentFile.write("<h3> Round Overview </h3>\n")
 323                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 324                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
 325                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
 326
 327                 agentFile.write("</table>\n")
 328
 329                 agentFile.write("<h3> Detailed </h3>\n")
 330                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 331                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
 332
 333                 for index in range(0, len(agent["ALL"])):
 334                         agentFile.write("<tr> <td> " + str(agent["ALL"][index][1]) + " </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
 335                 agentFile.write("</table>\n")
 336
 337                 agent["totalScore"] += agent["score"][0]
 338                 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
 339                 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
 340                 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
 341                 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
 342                 agent["Errors"] += len(agent["INTERNAL_ERROR"])
 343
 344                 agentFile.write("<h3> Accumulated Results </h3>\n")
 345                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 346                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
 347                 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
 348
 349                 agentFile.write("</table>\n")
 350                 agentFile.close()
 351
 352         #Update round file
 353         roundFile = open(htmlDir + "round"+str(roundNumber)+".html", "w")
 354         roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
 355         roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
 356         roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 357         roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
 358         agents.sort(key = lambda e : e["score"][0], reverse=True)
 359         for agent in agents:
 360                 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
 361         roundFile.write("</table>\n</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
 362         roundFile.close()
 363
 364
 365
 366
 367
 368 if verbose:
 369         print "Finalising .html files... "
 370 for agent in agents:
 371         agentFile = open(htmlDir + agent["name"]+".html", "a")
 372
 373         #Write the "total" statistics
 374
 375         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
 376         agentFile.close()
 377
 378         if os.path.exists(htmlDir + "total.html") == True:
 379                 os.remove(htmlDir + "total.html") #Delete the file
 380
 381 totalFile = open(htmlDir + "total.html", "w")
 382 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
 383 totalFile.write("<h1> Total Overview </h1>\n")
 384 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 385 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
 386 agents.sort(key = lambda e : e["totalScore"], reverse=True)
 387 for agent in agents:
 388         totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
 389 totalFile.write("</table>\n</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
 390 totalFile.close()
 391
 392
 393 if verbose:
 394         print "Done!"
 395
 396 endTime = time()
 397 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
 398 sys.exit(0)