icfp2015/filterResults.py

#!/usr/bin/env python
import json
import sys
import glob
from datetime import datetime

solutionsDir = "solutions/"

class bcolors:
	HEADER = '\033[95m'
	OKBLUE = '\033[94m'
	OKGREEN = '\033[92m'
	WARNING = '\033[93m'
	FAIL = '\033[91m'
	ENDC = '\033[0m'
	BOLD = '\033[1m'
	UNDERLINE = '\033[4m'

if len(sys.argv) < 2 or sys.argv[1] != "status":
	results = json.loads(''.join(sys.stdin.readlines()))

	resultsScore = {}
	with open('scores') as f:
		for line in f:
			tmp = line.strip().split(' ')
			key = tmp[0]+"."+tmp[1]
			if not key in resultsScore:
				resultsScore[key] = int(tmp[2])
			#else:
			#	print "Score alredy present for " + tmp[0] + " " + tmp[1] 
		f.close()

if len(sys.argv) < 2 or sys.argv[1] != "status":
	strategy = "unknown" if len(sys.argv) < 2 else sys.argv[1]
	for i in range(0, len(results)):
		key = str(results[i]['problemId']) + "." + str(results[i]['seed'])
		if key in resultsScore:
			out_file = open(solutionsDir+str(results[i]['problemId'])+"."+str(results[i]['seed'])+"."+"%.110d"%resultsScore[key]+"."+strategy,"w")
			results[i]['tag'] = strategy
			out_file.write(json.dumps(results[i]))
			out_file.close()
		else:
			print "Missing score for " + str(results[i]['problemId']) + " " + str(results[i]['seed'])
			sys.exit(1)
	problemIdForGlob = str(results[0]['problemId']) + '.'
else:
	problemIdForGlob = ''

solutionsToSend=[]
scores = {}
lastProblem = ""
for solution in sorted(glob.glob(solutionsDir + problemIdForGlob + '*'), reverse=True):
	problemHash = solution[:solution.rfind(".", 0, solution.rfind(".")) if solution.count('.') == 3 else solution.rfind(".")]
	if lastProblem != problemHash:
		lastProblem = problemHash
		with open(solution) as json_data:
			partialSolution = json.load(json_data)
			json_data.close()
		solutionsToSend.append(partialSolution)
		scores[lastProblem.replace(solutionsDir, '')] = int(solution.split('.')[2])
if len(sys.argv) < 2 or sys.argv[1] != "status":
	print json.dumps(list(reversed(solutionsToSend))).replace(', ', ',').replace(': ', ':')
else:
	import subprocess
	process = subprocess.Popen(['curl --user :2aaFCkjNIDHAsAIh9iQHc+Y+FGhkM5Z0RQgpO6TL6EA= -X GET https://davar.icfpcontest.org/teams/235/solutions'], shell = True, stdout=subprocess.PIPE)
	out, err = process.communicate()
	if err == None:
		serverResults = {}
		serverScores = {}
		results = json.loads(out)
		for result in results:
			#2015-08-08T12:26:42.894Z
			date_object = datetime.strptime(result['createdAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
#			print str(date_object)
			key = str(result['problemId'])+"."+str(result['seed'])
			if not key in serverResults or serverResults[key]['createdAt'] < result['createdAt']:
				serverResults[key] = result
			if not key in serverScores or serverScores[key] < result['score']:
				serverScores[key] = result
		serverBestResults = {}
		for result in results:
			key = str(result['problemId'])+"."+str(result['seed'])
			if serverScores[key]['score'] != None and result['score'] > serverScores[key]['score']:
				if not key in serverBestResults or serverBestResults[key] < result['score']:
					serverBestResults[key] = result
		print "Submitted solution: %d"%len(serverResults)
		for serverBestResultKey in serverBestResults:
			if serverBestResultKey in serverBestResults and serverBestResultKey in serverScores:
				if serverBestResults[serverBestResultKey]['score'] > serverScores[serverBestResultKey]['score']:
					print bcolors.WARNING + "a older solution with better score exists on server: %d %d %s"%(result['problemId'],result['seed'],result['solution']) + bcolors.ENDC
		tags = {}
		for serverResult in serverResults:
			if not serverResults[serverResult]['tag'] in tags:
				tags[serverResults[serverResult]['tag']] = 1
			else:
				tags[serverResults[serverResult]['tag']] += 1
		for solutionToSend in solutionsToSend:
			key = str(solutionToSend['problemId'])+"."+str(solutionToSend['seed'])
			if not key in serverResults:
				print bcolors.FAIL + str(solutionToSend['problemId'])+" "+str(solutionToSend['seed']) + " not present on server" + bcolors.ENDC
			else:
				if solutionToSend['solution'] != serverResults[key]['solution']:
					print bcolors.FAIL + str(solutionToSend['problemId'])+" "+str(solutionToSend['seed']) + " solution differ: " + solutionToSend['solution'] + " !=  " + serverResults[key]['solution'] + " (server)" + bcolors.ENDC
				elif serverResults[key]['score'] == None:
					found = False
					for i in range(1, len(results) + 1):
						result = results[len(results)- i]
						if result['problemId'] == solutionToSend['problemId'] and result['seed'] == solutionToSend['seed'] and result['solution'] == solutionToSend['solution'] and scores[key] == result['score']:
							found = True
							print bcolors.WARNING + str(solutionToSend['problemId'])+" "+str(solutionToSend['seed']) + " " + bcolors.OKGREEN + "score " + str(result['score']) + " ok" + bcolors.ENDC + bcolors.WARNING + ", but solution evaluation pending" + bcolors.ENDC
							break
					if not found:
						print bcolors.WARNING + str(solutionToSend['problemId'])+" "+str(solutionToSend['seed']) + " solution evaluation pending" + bcolors.ENDC
				elif scores[key] != serverResults[key]['score']:
					print bcolors.FAIL + str(solutionToSend['problemId'])+" "+str(solutionToSend['seed']) + " score differ: " + str(scores[key]) + " !=  " + str(serverResults[key]['score']) + " (server)" + bcolors.ENDC
				elif key in scores and serverScores[key]['score'] > scores[key]:
					print bcolors.FAIL + str(solutionToSend['problemId'])+" "+str(solutionToSend['seed']) + " better solution alredy exists on the server: " + serverScores[key]['solution'] + bcolors.ENDC
				elif not key in serverScores:
					print "cazziammolla"
		print "Tags statistics:"
		for tag in tags:
			print " %s: %d"%(tag, tags[tag])
#		print scores
#		print type(out)
#		print json.dumps(serverResults)
#		print len(serverResults)
	else:
		print "error retriving results from server"
		sys.exit(1)
sys.exit(0)