Related
I am trying to make a Tic-Tac-Toe game in Python using PyGame and the MiniMax algorithm. The AI plays really well when given the first chance (playing as 'X'), but becomes dumb enough to help the user win when not given the first chance (playing as 'O'). I think I know what the problem is but changing it is messing with the whole program and is not going by the given docstrings.
I've made two python files - one for the GUI (runner.py) and the other for the logic behind the game and the AI (tictactoe.py).
This is the logic behind the game:
# Import module `copy` for function `deepcopy` to deeply copy an
# original (mutable) object to save the object from mutations
import copy
X = 'X'
O = 'O'
EMPTY = None
def initial_state():
"""Returns starting state of the board
"""
return [
[EMPTY, EMPTY, EMPTY],
[EMPTY, EMPTY, EMPTY],
[EMPTY, EMPTY, EMPTY]
]
def display(board, autoprint=False):
"""Displays the board nested list as
a 3x3 matrix for board visualization
"""
vis_board = ''
for row in board:
for playr in row:
if playr is None:
playr = ' '
playr += ' '
vis_board += playr
vis_board += '\n'
if autoprint:
print(vis_board)
return vis_board
def player(board):
"""Returns player who has the next turn on a board
"""
global X, O
# Initial values for every call of the function
X_count = 0
O_count = 0
for row in board:
for playr in row:
if playr == X:
X_count += 1
elif playr == O:
O_count += 1
# `X` always starts first
if O_count < X_count:
return O
return X
def actions(board):
"""Returns set of all possible actions
(i, j) available on the board
"""
global EMPTY
action_set = set()
for i, row in enumerate(board):
for j, playr in enumerate(row):
if playr is EMPTY:
action_set.add((i, j))
return action_set
def result(board, action):
"""Returns the board that results from
making move (i, j) on the board.
"""
global EMPTY
if type(action) is not tuple or len(action) != 2:
raise Exception('invalid action taken')
# Using `deepcopy` to make a deepcopy of *board*
# as duplication by slicing entire list and by
# type conversion is not working poperly
dup_board = copy.deepcopy(board)
# Unpack the coordinates as `I` and `J`
I, J = action
# Check if place has not already been used
if dup_board[I][J] is EMPTY:
dup_board[I][J] = player(dup_board)
else:
raise Exception('invalid action taken')
return dup_board
def is_full(board):
"""Returns True if all places have been occupied, else returns False
"""
global EMPTY
for row in board:
for playr in row:
if playr is EMPTY:
return False
return True
def winner(board):
"""Returns the winner of the game, if there is one.
"""
winr = None # Initial declaration to avoid errors if no winner found
# Check diagonally
if (board[1][1] == board[0][0] and board[0][0] == board[2][2])\
or (board[1][1] == board[0][2] and board[0][2] == board[2][0]):
winr = board[1][1]
return winr
for i in range(3):
# Check each row for three-in-a-row
if board[i][0] == board[i][1] and board[i][1] == board[i][2]:
winr = board[i][1]
break
# Check each column for three-in-a-column
elif board[0][i] == board[1][i] and board[1][i] == board[2][i]:
winr = board[1][i]
break
return winr
def terminal(board):
"""Returns True if game is over, False otherwise.
"""
if winner(board) is None and not is_full(board):
return False
return True
def utility(board):
"""Returns 1 if X has won the game, -1 if O has won, 0 otherwise.
"""
global X, O
if terminal(board):
winr = winner(board)
if winr == X:
util = 1
elif winr == O:
util = -1
else:
util = 0
return util
return None
def get_best_score(board, is_max_turn):
"""Returns the best value of values of all possible moves
"""
if utility(board) is not None:
return utility(board)
scores = []
# Recursively help `minimax` choose the best action
# in `actions` of *board* by returning the best value
for action in actions(board):
rslt = result(board, action)
scores.append(get_best_score(rslt, not is_max_turn))
return max(scores) if is_max_turn else min(scores)
def minimax(board):
"""Returns the optimal action for the current player on the board.
"""
if terminal(board):
return None
best_score = -float('inf') # Least possible score
best_action = None
for action in actions(board):
rslt = result(board, action)
score = get_best_score(rslt, False)
if score > best_score:
best_score = score
best_action = action
return best_action
The GUI code file:
# Import module `PyGame` for a GUI
import pygame
import sys
import time
# Import module `tictactoe` (from the same folder as
# this file `__file__`) for the logic of the game's AI
import tictactoe as ttt
pygame.init()
size = width, height = 600, 400
# Colors
black = (0, 0, 0)
white = (255, 255, 255)
screen = pygame.display.set_mode(size)
mediumFont = pygame.font.Font('OpenSans-Regular.ttf', 24)
largeFont = pygame.font.Font('OpenSans-Regular.ttf', 40)
moveFont = pygame.font.Font('OpenSans-Regular.ttf', 60)
user = None
board = ttt.initial_state()
ai_turn = False
while True:
for event in pygame.event.get():
if event.type == pygame.QUIT:
sys.exit()
screen.fill(black)
# Let user choose a player.
if user is None:
# Draw title
title = largeFont.render('Play Tic-Tac-Toe', True, white)
titleRect = title.get_rect()
titleRect.center = (round(width/2), 50)
screen.blit(title, titleRect)
# Draw buttons
playXButton = pygame.Rect(round(width/8), round(height/2), round(width/4), 50)
playX = mediumFont.render('Play as X', True, black)
playXRect = playX.get_rect()
playXRect.center = playXButton.center
pygame.draw.rect(screen, white, playXButton)
screen.blit(playX, playXRect)
playOButton = pygame.Rect(5*round(width/8), round(height/2), round(width/4), 50)
playO = mediumFont.render('Play as O', True, black)
playORect = playO.get_rect()
playORect.center = playOButton.center
pygame.draw.rect(screen, white, playOButton)
screen.blit(playO, playORect)
# Check if button is clicked
click, _, _ = pygame.mouse.get_pressed()
if click == 1:
mouse = pygame.mouse.get_pos()
time.sleep(0.5)
if playXButton.collidepoint(mouse):
user = ttt.X
elif playOButton.collidepoint(mouse):
user = ttt.O
else:
# Draw game board
tile_size = 80
tile_origin = (width / 2 - (1.5 * tile_size),
height / 2 - (1.5 * tile_size))
tiles = []
for i in range(3):
row = []
for j in range(3):
rect = pygame.Rect(
round(tile_origin[0]+j*tile_size),
round(tile_origin[1]+i*tile_size),
round(tile_size), round(tile_size)
)
pygame.draw.rect(screen, white, rect, 3)
if board[i][j] != ttt.EMPTY:
move = moveFont.render(board[i][j], True, white)
moveRect = move.get_rect()
moveRect.center = rect.center
screen.blit(move, moveRect)
row.append(rect)
tiles.append(row)
game_over = ttt.terminal(board)
player = ttt.player(board)
# Show title
if game_over:
winner = ttt.winner(board)
if winner is None:
title = f'Game Over: Tie.'
else:
title = f'Game Over: {winner} wins.'
elif user == player:
title = f'Play as {user}'
else:
title = f'AI thinking...'
title = largeFont.render(title, True, white)
titleRect = title.get_rect()
titleRect.center = (round(width/2), 30)
screen.blit(title, titleRect)
# Check for AI move
if user != player and not game_over:
if ai_turn:
time.sleep(0.5)
move = ttt.minimax(board)
board = ttt.result(board, move)
ai_turn = False
else:
ai_turn = True
# Check for a user move
click, _, _ = pygame.mouse.get_pressed()
if click == 1 and user == player and not game_over:
mouse = pygame.mouse.get_pos()
for i in range(3):
for j in range(3):
if (board[i][j] == ttt.EMPTY and tiles[i][j].collidepoint(mouse)):
board = ttt.result(board, (i, j))
if game_over:
againButton = pygame.Rect(round(width/3), round(height-65), round(width/3), 50)
again = mediumFont.render('Play Again', True, black)
againRect = again.get_rect()
againRect.center = againButton.center
pygame.draw.rect(screen, white, againButton)
screen.blit(again, againRect)
click, _, _ = pygame.mouse.get_pressed()
if click == 1:
mouse = pygame.mouse.get_pos()
if againButton.collidepoint(mouse):
time.sleep(0.2)
user = None
board = ttt.initial_state()
ai_turn = False
pygame.display.flip()
These are the sidenotes for the answers given by the organization that gave these questions:
No changing the no. of parameters or the parameters themselves in any functions.
Follow the docstrings written in all functions
New functions may be defined as you wish
Please let me know if there are any bugs/errors which are causing the AI to be dumb when playing as 'O'. I believe the bug is in utility, but I can't change the code because it is not allowed (written in the docstrings).
Thank you!
Edit: The problem has been ALMOST solved, but the AI becomes dumb sometimes, like not trying to block the user's move with the opposite symbol, etc.
best_score = -float('inf') # Least possible score
you need to vary this according to the player for which you calculate the move. I think because of this the negative player is choosing random/first plausible move.
I have implemented minimax and related heuristics like 2 times, and always found that using the "negamax" approach worked best, since you don't need to worry about when to apply max and when min based on the player.
I am trying to make a Tic-Tac-Toe game in Python using PyGame and the MiniMax algorithm. The AI plays really well when given the first chance (playing as 'X'), but becomes dumb enough to help the user win when not given the first chance (playing as 'O'). I think I know what the problem is but changing it is messing with the whole program and is not going by the given docstrings.
I've made two python files - one for the GUI (runner.py) and the other for the logic behind the game and the AI (tictactoe.py).
This is the logic behind the game:
# Import module `copy` for function `deepcopy` to deeply copy an
# original (mutable) object to save the object from mutations
import copy
X = 'X'
O = 'O'
EMPTY = None
def initial_state():
"""Returns starting state of the board
"""
return [
[EMPTY, EMPTY, EMPTY],
[EMPTY, EMPTY, EMPTY],
[EMPTY, EMPTY, EMPTY]
]
def display(board, autoprint=False):
"""Displays the board nested list as
a 3x3 matrix for board visualization
"""
vis_board = ''
for row in board:
for playr in row:
if playr is None:
playr = ' '
playr += ' '
vis_board += playr
vis_board += '\n'
if autoprint:
print(vis_board)
return vis_board
def player(board):
"""Returns player who has the next turn on a board
"""
global X, O
# Initial values for every call of the function
X_count = 0
O_count = 0
for row in board:
for playr in row:
if playr == X:
X_count += 1
elif playr == O:
O_count += 1
# `X` always starts first
if O_count < X_count:
return O
return X
def actions(board):
"""Returns set of all possible actions
(i, j) available on the board
"""
global EMPTY
action_set = set()
for i, row in enumerate(board):
for j, playr in enumerate(row):
if playr is EMPTY:
action_set.add((i, j))
return action_set
def result(board, action):
"""Returns the board that results from
making move (i, j) on the board.
"""
global EMPTY
if type(action) is not tuple or len(action) != 2:
raise Exception('invalid action taken')
# Using `deepcopy` to make a deepcopy of *board*
# as duplication by slicing entire list and by
# type conversion is not working poperly
dup_board = copy.deepcopy(board)
# Unpack the coordinates as `I` and `J`
I, J = action
# Check if place has not already been used
if dup_board[I][J] is EMPTY:
dup_board[I][J] = player(dup_board)
else:
raise Exception('invalid action taken')
return dup_board
def is_full(board):
"""Returns True if all places have been occupied, else returns False
"""
global EMPTY
for row in board:
for playr in row:
if playr is EMPTY:
return False
return True
def winner(board):
"""Returns the winner of the game, if there is one.
"""
winr = None # Initial declaration to avoid errors if no winner found
# Check diagonally
if (board[1][1] == board[0][0] and board[0][0] == board[2][2])\
or (board[1][1] == board[0][2] and board[0][2] == board[2][0]):
winr = board[1][1]
return winr
for i in range(3):
# Check each row for three-in-a-row
if board[i][0] == board[i][1] and board[i][1] == board[i][2]:
winr = board[i][1]
break
# Check each column for three-in-a-column
elif board[0][i] == board[1][i] and board[1][i] == board[2][i]:
winr = board[1][i]
break
return winr
def terminal(board):
"""Returns True if game is over, False otherwise.
"""
if winner(board) is None and not is_full(board):
return False
return True
def utility(board):
"""Returns 1 if X has won the game, -1 if O has won, 0 otherwise.
"""
global X, O
if terminal(board):
winr = winner(board)
if winr == X:
util = 1
elif winr == O:
util = -1
else:
util = 0
return util
return None
def get_best_score(board, is_max_turn):
"""Returns the best value of values of all possible moves
"""
if utility(board) is not None:
return utility(board)
scores = []
# Recursively help `minimax` choose the best action
# in `actions` of *board* by returning the best value
for action in actions(board):
rslt = result(board, action)
scores.append(get_best_score(rslt, not is_max_turn))
return max(scores) if is_max_turn else min(scores)
def minimax(board):
"""Returns the optimal action for the current player on the board.
"""
if terminal(board):
return None
best_score = -float('inf') # Least possible score
best_action = None
for action in actions(board):
rslt = result(board, action)
score = get_best_score(rslt, False)
if score > best_score:
best_score = score
best_action = action
return best_action
The GUI code file:
# Import module `PyGame` for a GUI
import pygame
import sys
import time
# Import module `tictactoe` (from the same folder as
# this file `__file__`) for the logic of the game's AI
import tictactoe as ttt
pygame.init()
size = width, height = 600, 400
# Colors
black = (0, 0, 0)
white = (255, 255, 255)
screen = pygame.display.set_mode(size)
mediumFont = pygame.font.Font('OpenSans-Regular.ttf', 24)
largeFont = pygame.font.Font('OpenSans-Regular.ttf', 40)
moveFont = pygame.font.Font('OpenSans-Regular.ttf', 60)
user = None
board = ttt.initial_state()
ai_turn = False
while True:
for event in pygame.event.get():
if event.type == pygame.QUIT:
sys.exit()
screen.fill(black)
# Let user choose a player.
if user is None:
# Draw title
title = largeFont.render('Play Tic-Tac-Toe', True, white)
titleRect = title.get_rect()
titleRect.center = (round(width/2), 50)
screen.blit(title, titleRect)
# Draw buttons
playXButton = pygame.Rect(round(width/8), round(height/2), round(width/4), 50)
playX = mediumFont.render('Play as X', True, black)
playXRect = playX.get_rect()
playXRect.center = playXButton.center
pygame.draw.rect(screen, white, playXButton)
screen.blit(playX, playXRect)
playOButton = pygame.Rect(5*round(width/8), round(height/2), round(width/4), 50)
playO = mediumFont.render('Play as O', True, black)
playORect = playO.get_rect()
playORect.center = playOButton.center
pygame.draw.rect(screen, white, playOButton)
screen.blit(playO, playORect)
# Check if button is clicked
click, _, _ = pygame.mouse.get_pressed()
if click == 1:
mouse = pygame.mouse.get_pos()
time.sleep(0.5)
if playXButton.collidepoint(mouse):
user = ttt.X
elif playOButton.collidepoint(mouse):
user = ttt.O
else:
# Draw game board
tile_size = 80
tile_origin = (width / 2 - (1.5 * tile_size),
height / 2 - (1.5 * tile_size))
tiles = []
for i in range(3):
row = []
for j in range(3):
rect = pygame.Rect(
round(tile_origin[0]+j*tile_size),
round(tile_origin[1]+i*tile_size),
round(tile_size), round(tile_size)
)
pygame.draw.rect(screen, white, rect, 3)
if board[i][j] != ttt.EMPTY:
move = moveFont.render(board[i][j], True, white)
moveRect = move.get_rect()
moveRect.center = rect.center
screen.blit(move, moveRect)
row.append(rect)
tiles.append(row)
game_over = ttt.terminal(board)
player = ttt.player(board)
# Show title
if game_over:
winner = ttt.winner(board)
if winner is None:
title = f'Game Over: Tie.'
else:
title = f'Game Over: {winner} wins.'
elif user == player:
title = f'Play as {user}'
else:
title = f'AI thinking...'
title = largeFont.render(title, True, white)
titleRect = title.get_rect()
titleRect.center = (round(width/2), 30)
screen.blit(title, titleRect)
# Check for AI move
if user != player and not game_over:
if ai_turn:
time.sleep(0.5)
move = ttt.minimax(board)
board = ttt.result(board, move)
ai_turn = False
else:
ai_turn = True
# Check for a user move
click, _, _ = pygame.mouse.get_pressed()
if click == 1 and user == player and not game_over:
mouse = pygame.mouse.get_pos()
for i in range(3):
for j in range(3):
if (board[i][j] == ttt.EMPTY and tiles[i][j].collidepoint(mouse)):
board = ttt.result(board, (i, j))
if game_over:
againButton = pygame.Rect(round(width/3), round(height-65), round(width/3), 50)
again = mediumFont.render('Play Again', True, black)
againRect = again.get_rect()
againRect.center = againButton.center
pygame.draw.rect(screen, white, againButton)
screen.blit(again, againRect)
click, _, _ = pygame.mouse.get_pressed()
if click == 1:
mouse = pygame.mouse.get_pos()
if againButton.collidepoint(mouse):
time.sleep(0.2)
user = None
board = ttt.initial_state()
ai_turn = False
pygame.display.flip()
These are the sidenotes for the answers given by the organization that gave these questions:
No changing the no. of parameters or the parameters themselves in any functions.
Follow the docstrings written in all functions
New functions may be defined as you wish
Please let me know if there are any bugs/errors which are causing the AI to be dumb when playing as 'O'. I believe the bug is in utility, but I can't change the code because it is not allowed (written in the docstrings).
Thank you!
Edit: The problem has been ALMOST solved, but the AI becomes dumb sometimes, like not trying to block the user's move with the opposite symbol, etc.
best_score = -float('inf') # Least possible score
you need to vary this according to the player for which you calculate the move. I think because of this the negative player is choosing random/first plausible move.
I have implemented minimax and related heuristics like 2 times, and always found that using the "negamax" approach worked best, since you don't need to worry about when to apply max and when min based on the player.
I have written a Tic Tac Toe program that is working, but I am stuck on how to go about writing the chekDraw function (checking the game for a tie). All other functions are working, and I can play the game. My thought is that a for in range loop will go through the board indices, counting the x's and o's until the whole board is full. In main, I have the condition that if it is not a win (checkWin) then it is a tie. I have been staring at this so for so long, I would love fresh eyes on my code. Any insight/advice would be appreciated!
Edit: specifically what is happening with checkDraw is nothing- if the board is full and no winner, the game continually asks for a move, but any move at that point is illegal because all spots are taken (validation from getMove function).
# display instructions
def displayInstructions():
print()
print("This is a game of Tic-Tac-Toe. You will select an empty location")
print("and enter its index to select a move. The first player will be X")
print("and the second player will be O.")
print()
# display current state
# pass in board
# does not return anything
def showBoard(board):
for i in range(len(board)):
print("[" + str(board[i]) + "]", end="")
if i == 2 or i == 5 or i == 8:
print(end="\n")
# pass in board
# return updated board
# must validate move (in range, unoccupied square)
def getMove(board, player):
validMove = False
while not validMove:
move = input("{0}, what is your move: ".format(player))
position = int(move) - 1 # cast input as an integer, and puts player move in correct index
# in range
if position < 0 or position > 8:
print("That is an illegal move.")
# unoccupied square
if board[position] == "X" or board[position] == "O":
print("That is an illegal move.")
else:
# if valid move, put player on board
board[position] = player
return board
def checkWin(board, player):
if (board[0] == player and board[1] == player and board[2] == player) or \
(board[3] == player and board[4] == player and board[5] == player) or \
(board[6] == player and board[7] == player and board[8] == player) or \
(board[0] == player and board[3] == player and board[6] == player) or \
(board[1] == player and board[4] == player and board[7] == player) or \
(board[2] == player and board[5] == player and board[8] == player) or \
(board[0] == player and board[4] == player and board[8] == player) or \
(board[2] == player and board[4] == player and board[6] == player):
return True
else:
return False
def checkDraw(board, player):
count = 0
for i in range(len(board)):
if board[i] == player:
# if board[i] == "X" or board[i) == "O"
count += 1
if count == len(board):
return True
def main():
# Repeat play loop
playGame = True
while playGame:
# output instructions
displayInstructions()
# initialize board
board = [1, 2, 3, 4, 5, 6, 7, 8, 9]
print(len(board))
# initialize first player to X
player = "X"
# play until win or draw
while not checkWin(board, player) or checkDraw(board, player):
showBoard(board)
getMove(board, player)
checkWin(board, player)
checkDraw(board, player)
# if the game is in play (not a win or draw)
if not checkWin(board, player) or checkDraw(board, player):
# swap player
if player == "X":
player = "O"
else:
player = "X"
# if win
if checkWin(board, player):
showBoard(board)
print("{0} wins.".format(player))
playGame = False
# if draw
elif checkDraw(board, player):
showBoard(board)
print("It's a draw")
playGame = False
# Ask if want to play another game
playAgain = input("Would you like to play again? (y/n): ").lower()
if playAgain == "y":
main()
else:
print("Goodbye.")
if __name__ == "__main__":
main()
instead of these long series of and/or, you should build a list of board positions that form segments (lines) and use it as indirection to get all patterns on the board:
For example:
segments = [ (0,1,2),(3,4,5),(6,7,8),(0,3,6),(1,4,7),(2,5,8),(0,4,8),(2,4,6) ]
boardMarks = [ c if c in "XO" else "." for c in board ]
patterns = [ "".join(sorted(boardMarks[i] for i in s)) for s in segments ]
playerWins = (player*3) in patterns
xWins = "XXX" in patterns
oWins = "OOO" in patterns
draw = not xWins and not oWins and "." not in boardMarks
or, if you want to be more predictive and call out a draw early when there are no more moves that can win:
draw = not any(canWin in patterns for canWin in ("...","..X","..O",".XX",".OO"))
Because the patterns are sorted, there are only 5 combinations that represent a winnable line
If you want to make the computer play, you can use this 'autoPlay' function to select a position based on a mini-max rating system:
skill = 4 # 1 to 4 (novice to master)
def rating(position,player,board,level=skill+5):
if board[position] in "XO" or not level: return 0
newBoard = board[:position]+[player]+board[position+1:]
isWin = player*3 in ( "".join(newBoard[i] for i in s) for s in segments )
if isWin: return 3**level * 2
nextRatings = [rating(p,"XO"[player=="X"],newBoard,level-1) for p in range(9)]
return 3**level - max(nextRatings,key=abs)
from random import sample
def autoPlay(board,player):
return max(sample(range(9),9),key=lambda p:rating(p,player,board))
Example use:
position = autoPlay(board,player)
print("computer plays ",player," at position ",position+1)
board[position] = player
I'm making a game and it has different tiles obviously. But I've come to an issue in my main Game while loop.
def play():
player = player1()
while True:
room = ClubWorld.tile_at(player.x, player.y)
print(room.intro_text())
choose_action(room, player)
Example of a Tile in my game:
class GirlTile(MapTile):
def __init__(self,x,y):
self.meet_girl = Girls()
super().__init__(x, y)
def intro_text(self):
return "Hey whats up, my name is {}".format(self.meet_girl.name)
This loop keeps going as long as I'm on a game tile. It produces the available actions you have and lets the other functions know your position. it also outputs that tiles intro text, where my problem lies. I want the game to only output the intro text upon entry into a tile, once that happens i only want it to display the actions available. Suggestions?
You can keep previous_room and compare with room:
def play():
player = player1()
previous_room = None
while True:
room = ClubWorld.tile_at(player.x, player.y)
if room != previous_room:
print(room.intro_text())
previous_room = room
choose_action(room, player)
Or keep player previous position previous_x, previous_y and compare with new position
def play():
player = player1()
previous_x = None
previous_y = None
while True:
if player.x != previous_x or player.y != previous_y :
room = ClubWorld.tile_at(player.x, player.y)
print(room.intro_text())
previous_x = player.x
previous_y = player.y
choose_action(room, player)
I've written a program (listed below) which plays Tic Tic Toe with a Tkinter GUI. If I invoke it like this:
root = tk.Tk()
root.title("Tic Tac Toe")
player1 = QPlayer(mark="X")
player2 = QPlayer(mark="O")
human_player = HumanPlayer(mark="X")
player2.epsilon = 0 # For playing the actual match, disable exploratory moves
game = Game(root, player1=human_player, player2=player2)
game.play()
root.mainloop()
it works as expected and the HumanPlayer can play against player2, which is a computer player (specifically, a QPlayer). The figure below shows how the HumanPlayer (with mark "X") easily wins.
In order to improve the performance of the QPlayer, I'd like to 'train' it by allowing it to play against an instance of itself before playing against the human player. I've tried modifying the above code as follows:
root = tk.Tk()
root.title("Tic Tac Toe")
player1 = QPlayer(mark="X")
player2 = QPlayer(mark="O")
for _ in range(1): # Play a couple of training games
training_game = Game(root, player1, player2)
training_game.play()
training_game.reset()
human_player = HumanPlayer(mark="X")
player2.epsilon = 0 # For playing the actual match, disable exploratory moves
game = Game(root, player1=human_player, player2=player2)
game.play()
root.mainloop()
What I then find, however, is that the Tkinter window contains two Tic Tac Toe boards (depicted below), and the buttons of the second board are unresponsive.
In the above code, the reset() method is the same one as used in the callback of the "Reset" button, which usually makes the board blank again to start over. I don't understand why I'm seeing two boards (of which one is unresponsive) instead of a single, responsive board?
For reference, the full code of the Tic Tac Toe program is listed below (with the 'offensive' lines of code commented out):
import numpy as np
import Tkinter as tk
import copy
class Game:
def __init__(self, master, player1, player2, Q_learn=None, Q={}, alpha=0.3, gamma=0.9):
frame = tk.Frame()
frame.grid()
self.master = master
self.player1 = player1
self.player2 = player2
self.current_player = player1
self.other_player = player2
self.empty_text = ""
self.board = Board()
self.buttons = [[None for _ in range(3)] for _ in range(3)]
for i in range(3):
for j in range(3):
self.buttons[i][j] = tk.Button(frame, height=3, width=3, text=self.empty_text, command=lambda i=i, j=j: self.callback(self.buttons[i][j]))
self.buttons[i][j].grid(row=i, column=j)
self.reset_button = tk.Button(text="Reset", command=self.reset)
self.reset_button.grid(row=3)
self.Q_learn = Q_learn
self.Q_learn_or_not()
if self.Q_learn:
self.Q = Q
self.alpha = alpha # Learning rate
self.gamma = gamma # Discount rate
self.share_Q_with_players()
def Q_learn_or_not(self): # If either player is a QPlayer, turn on Q-learning
if self.Q_learn is None:
if isinstance(self.player1, QPlayer) or isinstance(self.player2, QPlayer):
self.Q_learn = True
def share_Q_with_players(self): # The action value table Q is shared with the QPlayers to help them make their move decisions
if isinstance(self.player1, QPlayer):
self.player1.Q = self.Q
if isinstance(self.player2, QPlayer):
self.player2.Q = self.Q
def callback(self, button):
if self.board.over():
pass # Do nothing if the game is already over
else:
if isinstance(self.current_player, HumanPlayer) and isinstance(self.other_player, HumanPlayer):
if self.empty(button):
move = self.get_move(button)
self.handle_move(move)
elif isinstance(self.current_player, HumanPlayer) and isinstance(self.other_player, ComputerPlayer):
computer_player = self.other_player
if self.empty(button):
human_move = self.get_move(button)
self.handle_move(human_move)
if not self.board.over(): # Trigger the computer's next move
computer_move = computer_player.get_move(self.board)
self.handle_move(computer_move)
def empty(self, button):
return button["text"] == self.empty_text
def get_move(self, button):
info = button.grid_info()
move = (info["row"], info["column"]) # Get move coordinates from the button's metadata
return move
def handle_move(self, move):
try:
if self.Q_learn:
self.learn_Q(move)
i, j = move # Get row and column number of the corresponding button
self.buttons[i][j].configure(text=self.current_player.mark) # Change the label on the button to the current player's mark
self.board.place_mark(move, self.current_player.mark) # Update the board
if self.board.over():
self.declare_outcome()
else:
self.switch_players()
except:
print "There was an error handling the move."
pass # This might occur if no moves are available and the game is already over
def declare_outcome(self):
if self.board.winner() is None:
print "Cat's game."
else:
print "The game is over. The player with mark %s won!" % self.current_player.mark
def reset(self):
print "Resetting..."
for i in range(3):
for j in range(3):
self.buttons[i][j].configure(text=self.empty_text)
self.board = Board(grid=np.ones((3,3))*np.nan)
self.current_player = self.player1
self.other_player = self.player2
# np.random.seed(seed=0) # Set the random seed to zero to see the Q-learning 'in action' or for debugging purposes
self.play()
def switch_players(self):
if self.current_player == self.player1:
self.current_player = self.player2
self.other_player = self.player1
else:
self.current_player = self.player1
self.other_player = self.player2
def play(self):
if isinstance(self.player1, HumanPlayer) and isinstance(self.player2, HumanPlayer):
pass # For human vs. human, play relies on the callback from button presses
elif isinstance(self.player1, HumanPlayer) and isinstance(self.player2, ComputerPlayer):
pass
elif isinstance(self.player1, ComputerPlayer) and isinstance(self.player2, HumanPlayer):
first_computer_move = player1.get_move(self.board) # If player 1 is a computer, it needs to be triggered to make the first move.
self.handle_move(first_computer_move)
elif isinstance(self.player1, ComputerPlayer) and isinstance(self.player2, ComputerPlayer):
while not self.board.over(): # Make the two computer players play against each other without button presses
move = self.current_player.get_move(self.board)
self.handle_move(move)
def learn_Q(self, move): # If Q-learning is toggled on, "learn_Q" should be called after receiving a move from an instance of Player and before implementing the move (using Board's "place_mark" method)
state_key = QPlayer.make_and_maybe_add_key(self.board, self.current_player.mark, self.Q)
next_board = self.board.get_next_board(move, self.current_player.mark)
reward = next_board.give_reward()
next_state_key = QPlayer.make_and_maybe_add_key(next_board, self.other_player.mark, self.Q)
if next_board.over():
expected = reward
else:
next_Qs = self.Q[next_state_key] # The Q values represent the expected future reward for player X for each available move in the next state (after the move has been made)
if self.current_player.mark == "X":
expected = reward + (self.gamma * min(next_Qs.values())) # If the current player is X, the next player is O, and the move with the minimum Q value should be chosen according to our "sign convention"
elif self.current_player.mark == "O":
expected = reward + (self.gamma * max(next_Qs.values())) # If the current player is O, the next player is X, and the move with the maximum Q vlue should be chosen
change = self.alpha * (expected - self.Q[state_key][move])
self.Q[state_key][move] += change
class Board:
def __init__(self, grid=np.ones((3,3))*np.nan):
self.grid = grid
def winner(self):
rows = [self.grid[i,:] for i in range(3)]
cols = [self.grid[:,j] for j in range(3)]
diag = [np.array([self.grid[i,i] for i in range(3)])]
cross_diag = [np.array([self.grid[2-i,i] for i in range(3)])]
lanes = np.concatenate((rows, cols, diag, cross_diag)) # A "lane" is defined as a row, column, diagonal, or cross-diagonal
any_lane = lambda x: any([np.array_equal(lane, x) for lane in lanes]) # Returns true if any lane is equal to the input argument "x"
if any_lane(np.ones(3)):
return "X"
elif any_lane(np.zeros(3)):
return "O"
def over(self): # The game is over if there is a winner or if no squares remain empty (cat's game)
return (not np.any(np.isnan(self.grid))) or (self.winner() is not None)
def place_mark(self, move, mark): # Place a mark on the board
num = Board.mark2num(mark)
self.grid[tuple(move)] = num
#staticmethod
def mark2num(mark): # Convert's a player's mark to a number to be inserted in the Numpy array representing the board. The mark must be either "X" or "O".
d = {"X": 1, "O": 0}
return d[mark]
def available_moves(self):
return [(i,j) for i in range(3) for j in range(3) if np.isnan(self.grid[i][j])]
def get_next_board(self, move, mark):
next_board = copy.deepcopy(self)
next_board.place_mark(move, mark)
return next_board
def make_key(self, mark): # For Q-learning, returns a 10-character string representing the state of the board and the player whose turn it is
fill_value = 9
filled_grid = copy.deepcopy(self.grid)
np.place(filled_grid, np.isnan(filled_grid), fill_value)
return "".join(map(str, (map(int, filled_grid.flatten())))) + mark
def give_reward(self): # Assign a reward for the player with mark X in the current board position.
if self.over():
if self.winner() is not None:
if self.winner() == "X":
return 1.0 # Player X won -> positive reward
elif self.winner() == "O":
return -1.0 # Player O won -> negative reward
else:
return 0.5 # A smaller positive reward for cat's game
else:
return 0.0 # No reward if the game is not yet finished
class Player(object):
def __init__(self, mark):
self.mark = mark
self.get_opponent_mark()
def get_opponent_mark(self):
if self.mark == 'X':
self.opponent_mark = 'O'
elif self.mark == 'O':
self.opponent_mark = 'X'
else:
print "The player's mark must be either 'X' or 'O'."
class HumanPlayer(Player):
def __init__(self, mark):
super(HumanPlayer, self).__init__(mark=mark)
class ComputerPlayer(Player):
def __init__(self, mark):
super(ComputerPlayer, self).__init__(mark=mark)
class RandomPlayer(ComputerPlayer):
def __init__(self, mark):
super(RandomPlayer, self).__init__(mark=mark)
#staticmethod
def get_move(board):
moves = board.available_moves()
if moves: # If "moves" is not an empty list (as it would be if cat's game were reached)
return moves[np.random.choice(len(moves))] # Apply random selection to the index, as otherwise it will be seen as a 2D array
class THandPlayer(ComputerPlayer):
def __init__(self, mark):
super(THandPlayer, self).__init__(mark=mark)
def get_move(self, board):
moves = board.available_moves()
if moves:
for move in moves:
if THandPlayer.next_move_winner(board, move, self.mark):
return move
elif THandPlayer.next_move_winner(board, move, self.opponent_mark):
return move
else:
return RandomPlayer.get_move(board)
#staticmethod
def next_move_winner(board, move, mark):
return board.get_next_board(move, mark).winner() == mark
class QPlayer(ComputerPlayer):
def __init__(self, mark, Q={}, epsilon=0.2):
super(QPlayer, self).__init__(mark=mark)
self.Q = Q
self.epsilon = epsilon
def get_move(self, board):
if np.random.uniform() < self.epsilon: # With probability epsilon, choose a move at random ("epsilon-greedy" exploration)
return RandomPlayer.get_move(board)
else:
state_key = QPlayer.make_and_maybe_add_key(board, self.mark, self.Q)
Qs = self.Q[state_key]
if self.mark == "X":
return QPlayer.stochastic_argminmax(Qs, max)
elif self.mark == "O":
return QPlayer.stochastic_argminmax(Qs, min)
#staticmethod
def make_and_maybe_add_key(board, mark, Q): # Make a dictionary key for the current state (board + player turn) and if Q does not yet have it, add it to Q
state_key = board.make_key(mark)
if Q.get(state_key) is None:
moves = board.available_moves()
Q[state_key] = {move: 0.0 for move in moves} # The available moves in each state are initially given a default value of zero
return state_key
#staticmethod
def stochastic_argminmax(Qs, min_or_max): # Determines either the argmin or argmax of the array Qs such that if there are 'ties', one is chosen at random
min_or_maxQ = min_or_max(Qs.values())
if Qs.values().count(min_or_maxQ) > 1: # If there is more than one move corresponding to the maximum Q-value, choose one at random
best_options = [move for move in Qs.keys() if Qs[move] == min_or_maxQ]
move = best_options[np.random.choice(len(best_options))]
else:
move = min_or_max(Qs, key=Qs.get)
return move
root = tk.Tk()
root.title("Tic Tac Toe")
player1 = QPlayer(mark="X")
player2 = QPlayer(mark="O")
# for _ in range(1): # Play a couple of training games
# training_game = Game(root, player1, player2)
# training_game.play()
# training_game.reset()
human_player = HumanPlayer(mark="X")
player2.epsilon = 0 # For playing the actual match, disable exploratory moves
game = Game(root, player1=human_player, player2=player2)
game.play()
root.mainloop()
It looks like you only need to create the board one time as the reset method resets it for the new players. Each type you create a Game instance, you create a new Tk frame so you either need to destroy the old one or you can reuse the windows by not creating a new Game instance each time.
A minor change to the main code at the bottom of the file seems to fix this:
player1 = QPlayer(mark="X")
player2 = QPlayer(mark="O")
game = Game(root, player1, player2)
for _ in range(1): # Play a couple of training games
game.play()
game.reset()
human_player = HumanPlayer(mark="X")
player2.epsilon = 0 # For playing the actual match, disable exploratory moves
game.player1 = human_player
game.player2 = player2
game.play()
I've noticed in this code that if you were to use it in python 3.2.3 or similar editions all of the print statements would need to be enclosed by brackets, and you'd need to add tkinter in the program by importing it.