应用CFR实现德州扑克对战

1
import random
2
from collections import defaultdict
3

4
CARDS = ['J', 'Q', 'K']
5

6
class KuhnCFR:
7
    def __init__(self):
8
        self.regret_sum = defaultdict(lambda: [0.0, 0.0])   # 每信息集两动作: [pass, bet]
9
        self.strategy_sum = defaultdict(lambda: [0.0, 0.0])
10

11
    def get_strategy(self, info_set):
12
        regrets = self.regret_sum[info_set]
13
        # Regret-Matching
14
        positive = [max(r, 0.0) for r in regrets]
15
        normalizer = sum(positive)
16
        if normalizer > 0:
17
            strategy = [r/normalizer for r in positive]
18
        else:
19
            strategy = [0.5, 0.5]
20
        # 累加平均策略
21
        for i in range(2):
22
            self.strategy_sum[info_set][i] += strategy[i]
23
        return strategy
24

25
    def get_average_strategy(self, info_set):
26
        s = self.strategy_sum[info_set]
27
        total = sum(s)
28
        if total > 0:
29
            return [x/total for x in s]
30
        return [0.5, 0.5]
31

32
    def cfr(self, cards, history, p0, p1):
33
        plays = len(history)
34
        player = plays % 2
35
        opponent = 1 - player
36
        # 终止判断
37
        if len(history) >= 2:
38
            terminal_pass = history[-1] == 'p'
39
            double_bet = history[-2:] == 'bb'
40
            if terminal_pass:
41
                # p 或 bp
42
                if history == 'pp':
43
                    return 1 if cards[player] > cards[opponent] else -1
44
                else:
45
                    return 1
46
            elif double_bet:
47
                return 2 if cards[player] > cards[opponent] else -2
48
        # 信息集：牌 + 历史
49
        info_set = cards[player] + history
50
        strategy = self.get_strategy(info_set)
51
        # 递归对手价值
52
        if player == 0:
53
            v_pass = -self.cfr(cards, history + 'p', p0*strategy[0], p1)
54
            v_bet  = -self.cfr(cards, history + 'b', p0*strategy[1], p1)
55
        else:
56
            v_pass = -self.cfr(cards, history + 'p', p0, p1*strategy[0])
57
            v_bet  = -self.cfr(cards, history + 'b', p0, p1*strategy[1])
58
        v = strategy[0]*v_pass + strategy[1]*v_bet
59
        # 反事实后悔
60
        regrets = [v_pass - v, v_bet - v]
61
        if player == 0:
62
            reach = p1
63
        else:
64
            reach = p0
65
        self.regret_sum[info_set][0] += reach * regrets[0]
66
        self.regret_sum[info_set][1] += reach * regrets[1]
67
        return v
68

69
    def train(self, iterations=100000):
70
        util = 0.0
71
        deck = CARDS[:]
72
        for _ in range(iterations):
73
            random.shuffle(deck)
74
            util += self.cfr(deck[:2], '', 1.0, 1.0)
75
        print(f"Average game value for Player 0: {util/iterations:.4f}")
76
        # 输出平均策略
77
        strat = {I: self.get_average_strategy(I) for I in self.strategy_sum}
78
        return strat
79

80
if __name__ == '__main__':
81
    solver = KuhnCFR()
82
    avg_strategy = solver.train(50000)
83
    for I in sorted(avg_strategy.keys()):
84
        s = avg_strategy[I]
85
        print(I, { 'pass': round(s[0],3), 'bet': round(s[1],3) })