progcomp/judge/simulator/simulate.py

   1 #!/usr/bin/python -u
   2
   3 '''
   4  simulate.py - simulation script for the 2012 UCC Programming Competition
   5         NOTE: This is not the manager program for a stratego game
   6         It merely calls the manager program as appropriate, and records results
   7         Plays exactly ONE round, but does not overwrite previously played rounds
   8         eg: run once to generate round1.results, twice to generate round2.results etc
   9         Also generates total.scores based on results from every round.
  10
  11         Now (sortof) generates .html files to display results in a prettiful manner.
  12
  13
  14  author Sam Moore (matches) [SZM]
  15  website http://matches.ucc.asn.au/stratego
  16  email [email protected] or [email protected]
  17  git git.ucc.asn.au/progcomp2012.git
  18 '''
  19
  20 import os
  21 import sys
  22 from time import time
  23
  24 #Global variables/arguments
  25
  26 baseDirectory = "../.." #Base directory for results, logs, agents
  27 nGames = 2 #Number of games played by each agent against each opponent. Half will be played as RED, half as BLUE. If nGames <= 1, then no games will be played (useful for dry run?)
  28 nRounds = 1
  29
  30 if len(sys.argv) >= 2:
  31         nRounds = int(sys.argv[1])
  32 if len(sys.argv) >= 3:
  33         nGames = int(sys.argv[2])
  34         if nGames % 2 != 0:
  35                 print "Warning: nGames should be even. "+str(nGames)+" specified, but only " + str(int(nGames/2) * 2)+" will be played!"
  36 if len(sys.argv) >= 4:
  37         baseDirectory = sys.argv[3]
  38 if len(sys.argv) >= 6:
  39         print "Useage: " +sys.argv[0] + " [nRounds=1] [nGames=10] [baseDirectory=\""+baseDirectory+"\"] [managerPath=baseDirectory+\"/judge/manager/stratego\"]"
  40         sys.exit(1)
  41
  42 resultsDirectory = baseDirectory+"/results/" #Where results will go (results are in the form of text files of agent names and scores)
  43 logDirectory = baseDirectory+"/log/" #Where log files go (direct output of manager program)
  44 agentsDirectory = baseDirectory+"/agents/" #Where agents are found (each agent has its own subdirectory within this directory)
  45 managerPath = baseDirectory+"/judge/manager/stratego" #Path to the executable that plays the games
  46 if len(sys.argv) >= 5:
  47         managerPath = sys.argv[5]
  48
  49
  50 #Score dictionary - Tuple is of the form: (end score, other score, other result) where end is the player on whose turn the result occurs, other is the other player, other result indicates what to record the outcome as for the other player.
  51 scores = {"VICTORY":(3,1, "DEFEAT"), "DEFEAT":(1,3, "VICTORY"), "SURRENDER":(1,3, "VICTORY"), "DRAW":(2,2, "DRAW"), "DRAW_DEFAULT":(1,1, "DRAW_DEFAULT"), "ILLEGAL":(-1,2, "DEFAULT"), "DEFAULT":(2,-1, "ILLEGAL"), "BOTH_ILLEGAL":(-1,-1, "BOTH_ILLEGAL"), "INTERNAL_ERROR":(0,0, "INTERNAL_ERROR"), "BAD_SETUP":(0,0,"BAD_SETUP")}
  52
  53
  54 #Verbose - print lots of useless stuff about what you are doing (kind of like matches talking on irc...)
  55 verbose = True
  56
  57
  58
  59 #Check the manager program exists TODO: And is executable!
  60 if os.path.exists(managerPath) == False:
  61         print "Manager program at \""+managerPath+"\" doesn't exist!"
  62         sys.exit(1)
  63
  64 #Make necessary directories
  65
  66 if os.path.exists(resultsDirectory) == False:
  67         os.mkdir(resultsDirectory) #Make the results directory if it didn't exist
  68 #Identify the round number by reading the results directory
  69 totalRounds = len(os.listdir(resultsDirectory)) + 1
  70 if totalRounds > 1:
  71         totalRounds -= 1
  72
  73 if os.path.exists(logDirectory) == False:
  74         os.mkdir(logDirectory) #Make the log directory if it didn't exist
  75
  76
  77 startTime = time() #Record time at which simulation starts
  78
  79 if verbose:
  80         if nRounds > 1:
  81                 print "Simulating " + str(nRounds) + " rounds (" + str(totalRounds) + " to " + str(totalRounds + nRounds-1) + ")"
  82         else:
  83                 print "Simulating one round."
  84         print ""
  85         print "Identifying possible agents in \""+agentsDirectory+"\""
  86
  87 #Get all agent names from agentsDirectory
  88 #TODO: Move this part outside the loop? It only has to happen once
  89 agentNames = os.listdir(agentsDirectory)
  90 agents = []
  91 for name in agentNames:
  92         if verbose:
  93                 sys.stdout.write("Scan \""+name+"\"... ")
  94         if os.path.isdir(agentsDirectory+name) == False: #Remove non-directories
  95                 if verbose:
  96                         sys.stdout.write(" Invalid! (Not a directory)\n")
  97                 continue
  98
  99         if os.path.exists(agentsDirectory+name+"/info") == False: #Try and find the special "info" file in each directory; ignore if it doesn't exist
 100                 if verbose:
 101                         sys.stdout.write(" Invalid! (No \"info\" file found)\n")
 102                 continue
 103
 104         agentExecutable = agentsDirectory+name+"/"+(open(agentsDirectory+name+"/info").readline().strip())
 105
 106         if os.path.exists(agentExecutable) == False:
 107                 if verbose:
 108                         sys.stdout.write(" Invalid! (Path: \""+agentExecutable+"\" does not exist!)\n")
 109                 continue
 110
 111
 112         if verbose:
 113                 sys.stdout.write(" Valid! (Path: \""+agentExecutable+"\")\n")
 114
 115         #Convert array of valid names into array of dictionaries containing information about each agent
 116         #I'm starting to like python...
 117         agents.append({"name":name, "path":agentExecutable,"score":[0], "totalScore":0, "VICTORY":[], "DEFEAT":[], "DRAW":[], "ILLEGAL":[], "INTERNAL_ERROR":[], "ALL":[]})
 118
 119 if len(agents) == 0:
 120         print "Couldn't find any agents! Check paths (Edit this script) or generate \"info\" files for agents."
 121         sys.exit(0)
 122 if verbose:
 123         print "Total: " + str(len(agents)) + " valid agents found (From "+str(len(agentNames))+" possibilities)"
 124         print ""
 125
 126 #Prepare the pretty .html files if they don't exist
 127 htmlDir = resultsDirectory + "pretty/"
 128 if os.path.exists(htmlDir) == False:
 129         os.mkdir(htmlDir)
 130 if os.path.exists(htmlDir) == False:
 131         print "Couldn't create directory \""+htmlDir+"\"."
 132         sys.exit(1)
 133
 134 for agent in agents:
 135         if os.path.exists(htmlDir+agent["name"] + ".html") == False:
 136                 agentFile = open(htmlDir+agent["name"] + ".html", "w")
 137                 agentFile.write("<html>\n<head>\n <title> " + agent["name"] + " results</title>\n</head>\n<body>\n<h1> Results for " + agent["name"]+" </h1>\n</body>\n</html>\n")
 138                 agentFile.close()
 139
 140         os.rename(htmlDir+agent["name"] + ".html", "tmpfile")
 141
 142         oldFile = open("tmpfile")
 143         agentFile = open(htmlDir+agent["name"] + ".html", "w")
 144         for line in oldFile:
 145                 if line.strip() == "</body>":
 146                         break
 147                 agentFile.write(line.strip() + "\n")
 148         oldFile.close()
 149         agentFile.close()
 150         os.remove("tmpfile")
 151
 152 #Do each round...
 153 totalGames = nGames/2 * len(agents) * (len(agents)-1)
 154 for roundNumber in range(totalRounds, totalRounds + nRounds):
 155
 156         if os.path.exists(logDirectory + "round"+str(roundNumber)) == False:
 157                 os.mkdir(logDirectory + "round"+str(roundNumber)) #Check there is a directory for this round's logs
 158
 159
 160         print "Commencing ROUND " + str(roundNumber) + " combat!"
 161         print "Total: " + str(totalGames) + " games to be played. This could take a while... (Estimate 60s/game)"
 162
 163
 164
 165         managerErrors = 0
 166         #This double for loop simulates a round robin, with each agent getting the chance to play as both red and blue against every other agent.
 167         gameNumber = 0
 168         for red in agents:  #for each agent playing as red,
 169                 for blue in agents: #against each other agent, playing as blue
 170                         if red == blue:
 171                                 continue #Exclude battles against self
 172                         gameNumber += 1
 173                         gameID = str(roundNumber) + "." + str(gameNumber)
 174                         for i in range(1, nGames/2 + 1):
 175                                 #Play a game and read the result. Note the game is logged to a file based on the agent's names
 176                                 if verbose:
 177                                         sys.stdout.write("Agents: \""+red["name"]+"\" and \""+blue["name"]+"\" playing game (ID: " + gameID + ") ... ")
 178                                 logFile = logDirectory + "round"+str(roundNumber) + "/"+red["name"]+".vs."+blue["name"]+"."+str(i)
 179                                 outline = os.popen(managerPath + " -o " + logFile + " " + red["path"] + " " + blue["path"], "r").read()
 180                                 results = outline.split(' ')
 181
 182                                 if len(results) != 6:
 183                                         if verbose:
 184                                                 sys.stdout.write("Garbage output! \"" + outline + "\"\n")
 185                                         red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
 186                                         blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
 187                                         red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
 188                                         blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR"))
 189                                         managerErrors += 1
 190                                 else:
 191
 192                                         if results[1] == "RED":
 193                                                 endColour = red
 194                                                 otherColour = blue
 195                                                 endStr = "RED"
 196                                                 otherStr = "BLUE"
 197                                         elif results[1] == "BLUE":
 198                                                 endColour = blue
 199                                                 otherColour = red
 200                                                 endStr = "BLUE"
 201                                                 otherStr = "RED"
 202
 203
 204                                         if results[1] == "BOTH":
 205                                                 red["INTERNAL_ERROR"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0]))
 206                                                 blue["INTERNAL_ERROR"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0]))
 207                                                 red["ALL"].append((blue["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "RED"))
 208                                                 blue["ALL"].append((red["name"], gameID, scores["INTERNAL_ERROR"][0], "INTERNAL_ERROR", "BLUE"))
 209                                                 managerErrors += 1
 210                                         else:
 211                                                 endColour["score"].insert(0,endColour["score"][0] + scores[results[2]][0])
 212                                                 endColour[results[2]].append((otherColour["name"], gameID, scores[results[2]][0]))
 213                                                 endColour["ALL"].append((otherColour["name"], gameID, scores[results[2]][0], results[2], endStr))
 214                                                 otherColour["score"].insert(0, otherColour["score"][0] + scores[results[2]][1])
 215                                                 otherColour[scores[results[2]][2]].append((endColour["name"], gameID, scores[results[2]][1]))
 216                                                 otherColour["ALL"].append((endColour["name"], gameID, scores[results[2]][1], scores[results[2]][2], otherStr))
 217
 218
 219                                         if verbose:
 220                                                 sys.stdout.write(" Result \"")
 221                                                 for ii in range(1, len(results)):
 222                                                         sys.stdout.write(results[ii].strip())
 223                                                         if ii < (len(results) - 1):
 224                                                                 sys.stdout.write(" ")
 225                                                 sys.stdout.write("\"\n")
 226
 227         if verbose:
 228                 print "Completed combat. Total of " + str(gameNumber) + " games played. "
 229         if managerErrors != 0:
 230                 print "WARNING: Registered "+str(managerErrors)+" errors. Check the manager program."
 231
 232         if verbose:
 233                 print ""
 234         #We should now have complete score values.
 235
 236         if verbose:
 237                 sys.stdout.write("Creating raw results files for ROUND " + str(roundNumber) + "... ")
 238
 239         agents.sort(key = lambda e : e["score"], reverse=True) #Sort the agents based on score
 240
 241         resultsFile = open(resultsDirectory+"round"+str(roundNumber)+".results", "w") #Create a file to store all the scores for this round
 242         for agent in agents:
 243                 resultsFile.write(agent["name"] + " " + str(agent["score"]) +"\n") #Write the agent names and scores into the file, in descending order
 244
 245         if verbose:
 246                 sys.stdout.write(" Complete!\n")
 247                 sys.stdout.write("Updating total scores... ");
 248
 249         #Now update the total scores
 250         if os.path.exists(resultsDirectory+"total.scores"):
 251                 if verbose:
 252                         sys.stdout.write(" Reading from \""+resultsDirectory+"total.scores\" to update scores... ")
 253                 totalFile = open(resultsDirectory+"total.scores", "r") #Try to open the total.scores file
 254                 for line in totalFile: #For all entries,
 255                         data = line.split(' ')
 256                         for agent in agents:
 257                                 if agent["name"] == data[0]:
 258                                         agent["totalScore"] = int(data[1]) + agent["score"][0] #Simply increment the current score by the recorded total score of the matching file entry
 259                                         break
 260                 totalFile.close() #Close the file, so we can delete it
 261                 os.remove(resultsDirectory+"total.scores") #Delete the file
 262                 #Sort the agents again
 263                 agents.sort(key = lambda e : e["totalScore"], reverse=True)
 264
 265         else:
 266                 if verbose:
 267                         sys.stdout.write(" First round - creating \""+resultsDirectory+"total.scores\"... ")
 268         if verbose:
 269                 sys.stdout.write(" Complete!\n")
 270                 print "Finished writing results for ROUND " + str(roundNumber)
 271                 print ""
 272
 273         if verbose:
 274                 print "RESULTS FOR ROUND " + str(roundNumber)
 275
 276         totalFile = open(resultsDirectory+"total.scores", "w") #Recreate the file
 277         for agent in agents:
 278                 totalFile.write(agent["name"] + " " + str(agent["totalScore"]) +"\n") #Write the total scores in descending order
 279                 if verbose:
 280                         print "Agent: " + str(agent)
 281
 282         if verbose:
 283                 print "Updating pretty .html files... "
 284
 285
 286
 287         for agent in agents:
 288                 agentFile = open(htmlDir + agent["name"]+".html", "a")
 289                 agentFile.write("<h2> Round " + str(roundNumber) + "</h2>\n")
 290                 agentFile.write("<h3> Summary </h3>\n")
 291                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 292                 agentFile.write("<tr> <th> Score </th> <th> Wins </th> <th> Losses </th> <th> Draws </th> <th> Illegal </th> <th> Errors </th></tr>\n")
 293                 agentFile.write("<tr> <td> "+str(agent["score"][0])+" </td> <td> "+str(len(agent["VICTORY"]))+" </td> <td> "+str(len(agent["DEFEAT"]))+" </td> <td> "+str(len(agent["DRAW"]))+" </td> <td> "+str(len(agent["ILLEGAL"]))+" </td> <td> " +str(len(agent["INTERNAL_ERROR"]))+" </td> </tr>\n")
 294
 295                 agentFile.write("</table>\n")
 296
 297                 agentFile.write("<h3> Detailed </h3>\n")
 298                 agentFile.write("<table border=\"0\" cellpadding=\"10\">\n")
 299                 agentFile.write("<tr> <th> Game ID </th> <th> Opponent </th> <th> Played as </th> <th> Outcome </th> <th> Score </th> <th> Accumulated Score </th> </tr> </th>\n")
 300
 301                 for index in range(0, len(agent["ALL"])):
 302                         agentFile.write("<tr> <td> " + str(agent["ALL"][index][1]) + " </td> <td> <a href="+agent["ALL"][index][0]+".html>"+agent["ALL"][index][0] + " </a> </td> <td> " + agent["ALL"][index][4] + " </td> <td> " + agent["ALL"][index][3] + " </td> <td> " + str(agent["ALL"][index][2]) + "</td> <td> " + str(agent["score"][len(agent["score"])-index -2]) + " </td> </tr> </th>\n")
 303                 agentFile.write("</table>\n")
 304                 agentFile.close()
 305
 306
 307 if verbose:
 308         print "Finalising .html files... "
 309 for agent in agents:
 310         agentFile = open(htmlDir + agent["name"]+".html", "a")
 311         agentFile.write("</body>\n<!-- Results file for \"" + agent["name"] + "\" autogenerated by \"" + sys.argv[0] + "\" at time " + str(time()) + " -->\n</html>\n\n")
 312         agentFile.close()
 313
 314
 315 if verbose:
 316         print "Done!"
 317
 318 endTime = time()
 319 print "Completed simulating " + str(nRounds) + " rounds in " + str(endTime - startTime) + " seconds."
 320 sys.exit(0)