Transformer简明教程 P8 定义数据

Transformer简明教程 P8 定义数据

字典就是拿一个数字代表一个词

代码：

# 定义字典
zidian_x = '<SOS>,<EOS>,<PAD>,0,1,2,3,4,5,6,7,8,9,q,w,e,r,t,y,u,i,o,p,a,s,d,f,g,h,j,k,l,z,x,c,v,b,n,m'
zidian_x = {word: i for i, word in enumerate(zidian_x.split(','))}
# print(zidian_x)zidian_xr = [k for k, v in zidian_x.items()]zidian_y = {k.upper(): v for k, v in zidian_x.items()}zidian_yr = [k for k, v in zidian_y.items()]import randomimport numpy as np
import torchdef get_data():# 定义词集合words = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'q', 'w', 'e', 'r','t', 'y', 'u', 'i', 'o', 'p', 'a', 's', 'd', 'f', 'g', 'h', 'j', 'k','l', 'z', 'x', 'c', 'v', 'b', 'n', 'm']# 定义每个词被选中的概率# m 概率最大p = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26])p = p / p.sum()# 随机选n个词# 随机数定义这个句子有多长（30-48之间）n = random.randint(30, 48)x = np.random.choice(words, size=n, replace=True, p=p)# 采样的结果就是xx = x.tolist()# y是对x的变换得到的# 字母大写,数字取10以内的互补数# 映射 求出ydef f(i):i = i.upper()if not i.isdigit():return ii = 9 - int(i)return str(i)# 双写 y的首字母 因为y 比 x 多一个y = [f(i) for i in x]y = y + [y[-1]]# 逆序y = y[::-1]# 加上首尾符号x = ['<SOS>'] + x + ['<EOS>']y = ['<SOS>'] + y + ['<EOS>']# 补pad到固定长度x = x + ['<PAD>'] * 50y = y + ['<PAD>'] * 51x = x[:50]y = y[:51]# 编码成数据x = [zidian_x[i] for i in x]y = [zidian_y[i] for i in y]# 转tensorx = torch.LongTensor(x)y = torch.LongTensor(y)return x, y# 定义数据集
class Dataset(torch.utils.data.Dataset):def __init__(self):super(Dataset, self).__init__()def __len__(self):return 100000def __getitem__(self, i):return get_data()# 数据加载器
# 每调用一次数据加载器得到的一批数据会有8个x和8个对应的y
loader = torch.utils.data.DataLoader(dataset=Dataset(),batch_size=8,drop_last=True,shuffle=True,collate_fn=None)