GPT আর্কিটেকচার

BERT বনাম GPT

Causal Mask

# Future token দেখতে দেয় না
mask = torch.triu(torch.full((T, T), float("-inf")), diagonal=1)

scores = (Q @ K.T) / d_k**0.5 + mask
attn = scores.softmax(-1) @ V

GPT Block (Pre-LN)

class GPTBlock(nn.Module):
    def __init__(self, d, h, ff):
        super().__init__()
        self.ln1 = nn.LayerNorm(d)
        self.attn = nn.MultiheadAttention(d, h, batch_first=True)
        self.ln2 = nn.LayerNorm(d)
        self.mlp = nn.Sequential(nn.Linear(d, ff), nn.GELU(), nn.Linear(ff, d))
    def forward(self, x, mask):
        a,_ = self.attn(self.ln1(x), self.ln1(x), self.ln1(x), attn_mask=mask)
        x = x + a
        x = x + self.mlp(self.ln2(x))
        return x

সম্পূর্ণ Tiny-GPT

class TinyGPT(nn.Module):
    def __init__(self, vocab, d=256, h=8, L=6, T=256):
        super().__init__()
        self.tok = nn.Embedding(vocab, d)
        self.pos = nn.Embedding(T, d)
        self.blocks = nn.ModuleList([GPTBlock(d, h, 4*d) for _ in range(L)])
        self.ln = nn.LayerNorm(d)
        self.head = nn.Linear(d, vocab, bias=False)
        self.T = T
    def forward(self, idx):
        B, T = idx.shape
        mask = torch.triu(torch.full((T,T), float("-inf"),
                                     device=idx.device), 1)
        x = self.tok(idx) + self.pos(torch.arange(T, device=idx.device))
        for blk in self.blocks: x = blk(x, mask)
        return self.head(self.ln(x))   # (B, T, vocab)

Sampling Strategies

• Greedy: সবসময় argmax — পুনরাবৃত্তি।
• Temperature: logits/T — কম T = sharp, বেশি T = creative।
• Top-k: শীর্ষ k token থেকে sample।
• Top-p (nucleus): cumulative prob p পর্যন্ত।
• Beam search: একাধিক sequence সমান্তরালে — translation।

def generate(model, idx, max_new, T=1.0, top_k=50):
    for _ in range(max_new):
        x = idx[:, -model.T:]
        logits = model(x)[:, -1] / T
        if top_k:
            v, _ = logits.topk(top_k)
            logits[logits < v[:, [-1]]] = -float("inf")
        probs = logits.softmax(-1)
        nxt = torch.multinomial(probs, 1)
        idx = torch.cat([idx, nxt], 1)
    return idx

GPT পরিবার-এর ক্রমবিকাশ

HuggingFace দিয়ে generation

from transformers import AutoTokenizer, AutoModelForCausalLM
tok = AutoTokenizer.from_pretrained("gpt2")
mdl = AutoModelForCausalLM.from_pretrained("gpt2")

ids = tok("Deep learning is", return_tensors="pt").input_ids
out = mdl.generate(ids, max_new_tokens=40, do_sample=True,
                   top_p=0.9, temperature=0.8)
print(tok.decode(out[0], skip_special_tokens=True))

অনুশীলন

সারসংক্ষেপ