judge/simulator/simulate.py

   1 #!/usr/bin/python -u
   2
   3 '''
   4  simulate.py - simulation script for the 2012 UCC Programming Competition
   5         NOTE: This is not the manager program for a stratego game
   6         It merely calls the manager program as appropriate, and records results
   7         Plays exactly ONE round, but does not overwrite previously played rounds
   8         eg: run once to generate round1.results, twice to generate round2.results etc
   9         Also generates total.scores based on results from every round.
  10
  11         Now (sortof) generates .html files to display results in a prettiful manner.
  12
  13
  14  author Sam Moore (matches) [SZM]
  15  website http://matches.ucc.asn.au/stratego
  16  email [email protected] or [email protected]
  17  git git.ucc.asn.au/progcomp2012.git
  18 '''
  19
  20 import os
  21 import sys
  22 from time import time
  23
  24 #Global variables/arguments
  25
  26 baseDirectory = "../.." #Base directory for results, logs, agents
  27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
  28 nRounds = 1
  29
  30 timeoutValue = 2
  31
  32 if len(sys.argv) >= 2:
  33         nRounds = int(sys.argv[1])
  34 if len(sys.argv) >= 3:
  35         nGames = int(sys.argv[2])
  36         if nGames % 2 != 0:
  37                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
  38 if len(sys.argv) >= 4:
  39         baseDirectory = sys.argv[3]
  40 if len(sys.argv) >= 6:
  41         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
  42         sys.exit(1)
  43
  44 resultsDirectory = baseDirectory+"/web/results/" #Where results will go (results are in the form of text files of agent names and scores)
  45 logDirectory = baseDirectory+"/web/log/" #Where log files go (direct output of manager program)
  46 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
  47 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
  48 if len(sys.argv) >= 5:
  49         managerPath = sys.argv[5]
  50
  51
  52 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
  53 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
  54
  55
  56 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
  57 verbose = True
  58
  59
  60
  61 #Check the manager program exists TODO: And is executable!
  62 if os.path.exists(managerPath) == False:
  63         print "Manager program at \""+managerPath+"\" doesn't exist!"
  64         sys.exit(1)
  65
  66 #Make necessary directories
  67
  68 if os.path.exists(resultsDirectory) == False:
  69         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
  70
  71
  72 #Identify the round number by reading from the "info" file in the results directory, if it doesn't exist then start at round 1.
  73 if os.path.exists(resultsDirectory+"info") == False:
  74         totalRounds = 1
  75 else:
  76         info = open(resultsDirectory+"info", "r")
  77         totalRounds = int(info.readline().strip())
  78         info.close()
  79         os.remove(resultsDirectory+"info")
  80
  81 info = open(resultsDirectory+"info", "w")
  82 info.write(str(totalRounds + nRounds) + "\n")
  83 info.close()
  84
  85
  86
  87 if os.path.exists(logDirectory) == False:
  88         os.mkdir(logDirectory) #Make the log directory if it didn't exist
  89
  90
  91 startTime = time() #Record time at which simulation starts
  92
  93 if verbose:
  94         if nRounds > 1:
  95                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
  96         else:
  97                 print "Simulating one round."
  98         print ""
  99         print "Identifying possible agents in \""+agentsDirectory+"\""
 100
 101 #Get all agent names from agentsDirectory
 102 agentNames = os.listdir(agentsDirectory)
 103 agents = []
 104 for name in agentNames:
 105         if verbose:
 106                 sys.stdout.write("Scan \""+name+"\"... ")
 107         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
 108                 if verbose:
 109                         sys.stdout.write(" Invalid! (Not a directory)\n")
 110                 continue
 111
 112         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist
 113                 if verbose:
 114                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
 115                 continue
 116
 117         infoFile = open(agentsDirectory+name+"/info", "r")
 118         agentExecutable = agentsDirectory+name+"/"+(infoFile.readline().strip())
 119         author = infoFile.readline().strip()
 120         language = infoFile.readline().strip()
 121         description = ""
 122         while True:
 123                 line = infoFile.readline()
 124                 if len(line) > 0:
 125                         description += line
 126                 else:
 127                         break
 128         infoFile.close()
 129
 130         if os.path.exists(agentExecutable.split(" ")[0]) == False:
 131                 if verbose:
 132                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
 133                 continue
 134
 135
 136         if verbose:
 137                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
 138
 139         #Convert array of valid names into array of dictionaries containing information about each agent
 140         #I'm starting to like python...
 141         agents.append({"name":name, "path":agentExecutable, "author":author, "language":language, "description":description, "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[], "totalScore":0, "Wins":0, "Losses":0, "Draws":0, "Illegal":0, "Errors":0})
 142
 143 if len(agents) == 0:
 144         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
 145         sys.exit(0)
 146 if verbose:
 147         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
 148         print ""
 149
 150 #Prepare the pretty .html files if they don't exist
 151 if verbose:
 152         print "Preparing .html results files..."
 153
 154
 155 for agent in agents:
 156         if os.path.exists(resultsDirectory+agent["name"] + ".html") == False:
 157                 agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
 158                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " overview</title>\n</head>\n<body>\n<h1> Overview for " + agent["name"]+" </h1>\n")
 159                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 160                 agentFile.write("<tr> <th> Name </th> <th> Author </th> <th> Language </th> </tr>\n")
 161                 agentFile.write("<tr> <td> "+agent["name"]+" </td> <td> "+agent["author"]+" </td> <td> "+agent["language"]+" </td> </tr>\n")
 162                 agentFile.write("</table>\n");
 163
 164                 agentFile.write("<p> <b>Description</b> </p>\n")
 165                 agentFile.write("<p> " + agent["description"] + " </p>\n")
 166                 agentFile.close()
 167
 168         os.rename(resultsDirectory+agent["name"] + ".html", "tmpfile")
 169
 170         oldFile = open("tmpfile", "r")
 171         agentFile = open(resultsDirectory+agent["name"] + ".html", "w")
 172         line = oldFile.readline()
 173         while line != "":
 174                 #if verbose:
 175                 #       print "Interpreting line \"" + line.strip() + "\""
 176                 if line.strip() == "</body>":
 177                         break
 178                 elif line == "<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n":
 179                         agentFile.write(line)
 180                         line = oldFile.readline()
 181
 182                         values = line.split(' ')
 183                         agent["totalScore"] += int(values[2].strip())
 184                         agent["Wins"] += int(values[5].strip())
 185                         agent["Losses"] += int(values[8].strip())
 186                         agent["Draws"] += int(values[11].strip())
 187                         agent["Illegal"] += int(values[14].strip())
 188                         agent["Errors"] += int(values[17].strip())
 189                 agentFile.write(line)
 190                 line = oldFile.readline()
 191
 192         if verbose:
 193                 print "Prepared results file \"" + resultsDirectory+agent["name"] + ".html\"."
 194         oldFile.close()
 195         agentFile.close()
 196         os.remove("tmpfile")
 197
 198 if verbose:
 199         print ""
 200
 201 #Do each round...
 202 totalGames = nGames/2 * len(agents) * (len(agents)-1)
 203 for roundNumber in range(totalRounds, totalRounds + nRounds):
 204
 205         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
 206                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
 207
 208         for agent in agents:
 209                 agent.update({"name":agent["name"], "path":agent["path"],  "score":[0], "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "DEFAULT":[], "INTERNAL_ERROR":[], "SURRENDER":[], "DRAW_DEFAULT":[], "BOTH_ILLEGAL":[], "BAD_SETUP":[], "ALL":[]})
 210
 211
 212         print "Commencing ROUND " + str(roundNumber) + " combat!"
 213         print "Total: " + str(totalGames) + " games to be played. This could take a while..."
 214
 215
 216
 217         managerErrors = 0
 218         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
 219         gameNumber = 0
 220         for red in agents:  #for each agent playing as red,
 221                 for blue in agents: #against each other agent, playing as blue
 222                         if red == blue:
 223                                 continue #Exclude battles against self
 224
 225
 226                         for i in range(1, nGames/2 + 1):
 227                                 gameNumber += 1
 228                                 gameID = str(roundNumber) + "." + str(gameNumber)
 229                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
 230                                 if verbose:
 231                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
 232                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(gameID)
 233                                 errorLog = [logDirectory + "error/" + red["name"] + "."+str(gameID), logDirectory + "error/" + blue["name"] + "."+str(gameID)]
 234                                 #Run the game, outputting to logFile; stderr of (both) AI programs is directed to logFile.stderr
 235                                 outline = os.popen(managerPath + " -o " + logFile + " -T " + str(timeoutValue) + " \"" + red["path"] + "\" \"" + blue["path"] + "\" 2>> " + logFile+".stderr", "r").read()
 236
 237                                 #If there were no errors, get rid of the stderr file
 238                                 if os.stat(logFile+".stderr").st_size <= 0:
 239                                         os.remove(logFile+".stderr")
 240                                 results = outline.split(' ')
 241
 242                                 if len(results) != 6:
 243                                         if verbose:
 244                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
 245                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
 246                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
 247                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
 248                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
 249                                         managerErrors += 1
 250                                 else:
 251
 252                                         if results[1] == "RED":
 253                                                 endColour = red
 254                                                 otherColour = blue
 255                                                 endStr = "RED"
 256                                                 otherStr = "BLUE"
 257                                         elif results[1] == "BLUE":
 258                                                 endColour = blue
 259                                                 otherColour = red
 260                                                 endStr = "BLUE"
 261                                                 otherStr = "RED"
 262
 263
 264                                         if results[1] == "BOTH":
 265                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
 266                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
 267                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
 268                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
 269                                                 managerErrors += 1
 270                                         else:
 271                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
 272                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
 273                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
 274                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
 275                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
 276                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
 277
 278
 279                                         if verbose:
 280                                                 sys.stdout.write(" Result \"")
 281                                                 for ii in range(1, len(results)):
 282                                                         sys.stdout.write(results[ii].strip())
 283                                                         if ii < (len(results) - 1):
 284                                                                 sys.stdout.write(" ")
 285                                                 sys.stdout.write("\"\n")
 286
 287         if verbose:
 288                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
 289         if managerErrors != 0:
 290                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
 291
 292         if verbose:
 293                 print ""
 294         #We should now have complete score values.
 295         if verbose:
 296                 print "RESULTS FOR ROUND " + str(roundNumber)
 297
 298         #totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
 299                 #for agent in agents:
 300                 #totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
 301                 #if verbose:
 302                 #               print "Agent: " + str(agent)
 303
 304
 305         if verbose:
 306                 print "Updating pretty .html files... "
 307
 308         for agent in agents:
 309                 agentFile = open(resultsDirectory + agent["name"]+".html", "a")
 310                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
 311                 agentFile.write("<h3> Round Overview </h3>\n")
 312                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 313                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
 314                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]) + len(agent["DEFAULT"]))+" </td> <td> "+str(len(agent["DEFEAT"]) + len(agent["SURRENDER"]))+" </td> <td> "+str(len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"]))+" </td> <td> "+str(len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
 315
 316                 agentFile.write("</table>\n")
 317                 agentFile.write("<p> <a href=round"+str(roundNumber)+".html>Round "+str(roundNumber) + " Scoreboard</a></p>\n")
 318
 319                 agentFile.write("<h3> Detailed </h3>\n")
 320                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 321                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
 322
 323
 324
 325                 for index in range(0, len(agent["ALL"])):
 326                         if agent["ALL"][index][4] == "RED":
 327                                 logFile = "../log/round"+str(roundNumber) + "/"+agent["name"]+".vs."+agent["ALL"][index][0]+"."+str(agent["ALL"][index][1])
 328                         else:
 329                                 logFile = "../log/round"+str(roundNumber) + "/"+agent["ALL"][index][0]+".vs."+agent["name"]+"."+str(agent["ALL"][index][1])
 330                         agentFile.write("<tr> <td> <a href="+logFile+">" + str(agent["ALL"][index][1]) + " </a> </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
 331                 agentFile.write("</table>\n")
 332
 333                 agent["totalScore"] += agent["score"][0]
 334                 agent["Wins"] += len(agent["VICTORY"]) + len(agent["DEFAULT"])
 335                 agent["Losses"] += len(agent["DEFEAT"]) + len(agent["SURRENDER"])
 336                 agent["Draws"] += len(agent["DRAW"]) + len(agent["DRAW_DEFAULT"])
 337                 agent["Illegal"] += len(agent["ILLEGAL"]) + len(agent["BOTH_ILLEGAL"]) + len(agent["BAD_SETUP"])
 338                 agent["Errors"] += len(agent["INTERNAL_ERROR"])
 339
 340                 agentFile.write("<h3> Accumulated Results </h3>\n")
 341                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 342                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
 343                 agentFile.write("<tr> <td> "+str(agent["totalScore"])+" </td> <td> "+str(agent["Wins"])+" </td> <td> "+str(agent["Losses"])+" </td> <td> "+str(agent["Draws"])+" </td> <td> "+str(agent["Illegal"])+" </td> <td> " +str(agent["Errors"])+" </td> </tr>\n")
 344
 345                 agentFile.write("</table>\n")
 346
 347
 348                 agentFile.close()
 349
 350         #Update round file
 351         roundFile = open(resultsDirectory + "round"+str(roundNumber)+".html", "w")
 352         roundFile.write("<html>\n<head>\n <title> Round " +str(roundNumber)+ " Overview </title>\n</head>\n<body>\n")
 353         roundFile.write("<h1> Round " +str(roundNumber)+ " Overview </h1>\n")
 354         roundFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 355         roundFile.write("<tr> <th> Name </th> <th> Score </th> <th> Total Score </th> </tr>\n")
 356         agents.sort(key = lambda e : e["score"][0], reverse=True)
 357         for agent in agents:
 358                 roundFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["score"][0]) + " </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
 359         roundFile.write("</table>\n")
 360         roundFile.write("<p> <a href=total.html>Current Scoreboard</a></p>\n")
 361         roundFile.write("</body>\n<!-- Results file for Round " + str(roundNumber) + " autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
 362         roundFile.close()
 363
 364
 365
 366
 367
 368 if verbose:
 369         print "Finalising .html files... "
 370 for agent in agents:
 371         agentFile = open(resultsDirectory + agent["name"]+".html", "a")
 372
 373         #Write the "total" statistics
 374
 375         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
 376         agentFile.close()
 377
 378         if os.path.exists(resultsDirectory + "total.html") == True:
 379                 os.remove(resultsDirectory + "total.html") #Delete the file
 380
 381 totalFile = open(resultsDirectory + "total.html", "w")
 382 totalFile.write("<html>\n<head>\n <title> Total Overview </title>\n</head>\n<body>\n")
 383 totalFile.write("<h1> Total Overview </h1>\n")
 384 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 385 totalFile.write("<tr> <th> Name </th> <th> Total Score </th> </tr>\n")
 386 agents.sort(key = lambda e : e["totalScore"], reverse=True)
 387 for agent in agents:
 388         totalFile.write("<tr> <td> <a href="+agent["name"]+".html>"+agent["name"] + " </a> </td> <td> " + str(agent["totalScore"]) + " </td> </tr>\n")
 389 totalFile.write("</table>\n")
 390
 391 totalFile.write("<h2> Round Summaries </h2>\n")
 392 totalFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 393 for i in range(1, totalRounds+1):
 394         totalFile.write("<tr> <td> <a href=round"+str(i)+".html>Round " + str(i) + "</a> </td> </tr>\n")
 395 totalFile.write("</table>\n")
 396
 397 totalFile.write("</body>\n<!-- Total Results file autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
 398 totalFile.close()
 399
 400
 401 if verbose:
 402         print "Done!"
 403
 404 endTime = time()
 405 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
 406 sys.exit(0)