阅读量:1
这节课主要学习基于MindSpore的GPT2文本摘要。主要包括环境安装、数据集加载与处理、模型构建、模型训练、模型推理五部分内容。
1.首先介绍环境安装
%%capture captured_output # 实验环境已经预装了mindspore==2.2.14，如需更换mindspore版本，可更改下面mindspore的版本号 !pip uninstall mindspore -y !pip install -i https://pypi.mirrors.ustc.edu.cn/simple mindspore==2.2.14 !pip install tokenizers==0.15.0 -i https://pypi.tuna.tsinghua.edu.cn/simple # 该案例在 mindnlp 0.3.1 版本完成适配，如果发现案例跑不通，可以指定mindnlp版本，执行`!pip install mindnlp==0.3.1` !pip install mindnlp 搭建mindspore深度学习环境。
2.数据集加载与处理
2.1 数据集加载
实验使用的是nlpcc2017摘要数据，内容为新闻正文及其摘要，总计50000个样本。
from mindnlp.utils import http_get # download dataset url = 'https://download.mindspore.cn/toolkits/mindnlp/dataset/text_generation/nlpcc2017/train_with_summ.txt' path = http_get(url, './') from mindspore.dataset import TextFileDataset # load dataset dataset = TextFileDataset(str(path), shuffle=False) dataset.get_dataset_size() # split into training and testing dataset train_dataset, test_dataset = dataset.split([0.9, 0.1], randomize=False) 2.2 数据预处理
原始数据格式：
article: [CLS] article_context [SEP]
summary: [CLS] summary_context [SEP]
预处理后的数据格式：
[CLS] article_context [SEP] summary_context [SEP]
import json import numpy as np # preprocess dataset def process_dataset(dataset, tokenizer, batch_size=6, max_seq_len=1024, shuffle=False): def read_map(text): data = json.loads(text.tobytes()) return np.array(data['article']), np.array(data['summarization']) def merge_and_pad(article, summary): # tokenization # pad to max_seq_length, only truncate the article tokenized = tokenizer(text=article, text_pair=summary, padding='max_length', truncation='only_first', max_length=max_seq_len) return tokenized['input_ids'], tokenized['input_ids'] dataset = dataset.map(read_map, 'text', ['article', 'summary']) # change column names to input_ids and labels for the following training dataset = dataset.map(merge_and_pad, ['article', 'summary'], ['input_ids', 'labels']) dataset = dataset.batch(batch_size) if shuffle: dataset = dataset.shuffle(batch_size) return dataset 因GPT2无中文的tokenizer，我们使用BertTokenizer替代。
from mindnlp.transformers import BertTokenizer # We use BertTokenizer for tokenizing chinese context. tokenizer = BertTokenizer.from_pretrained('bert-base-chinese') len(tokenizer) train_dataset = process_dataset(train_dataset, tokenizer, batch_size=4) next(train_dataset.create_tuple_iterator()) 3.模型构建
3.1构建GPT2ForSummarization模型，注意shift right的操作。
from mindspore import ops from mindnlp.transformers import GPT2LMHeadModel class GPT2ForSummarization(GPT2LMHeadModel): def construct( self, input_ids = None, attention_mask = None, labels = None, ): outputs = super().construct(input_ids=input_ids, attention_mask=attention_mask) shift_logits = outputs.logits[..., :-1, :] shift_labels = labels[..., 1:] # Flatten the tokens loss = ops.cross_entropy(shift_logits.view(-1, shift_logits.shape[-1]), shift_labels.view(-1), ignore_index=tokenizer.pad_token_id) return loss 3.2.动态学习率
from mindspore import ops from mindspore.nn.learning_rate_schedule import LearningRateSchedule class LinearWithWarmUp(LearningRateSchedule): """ Warmup-decay learning rate. """ def __init__(self, learning_rate, num_warmup_steps, num_training_steps): super().__init__() self.learning_rate = learning_rate self.num_warmup_steps = num_warmup_steps self.num_training_steps = num_training_steps def construct(self, global_step): if global_step < self.num_warmup_steps: return global_step / float(max(1, self.num_warmup_steps)) * self.learning_rate return ops.maximum( 0.0, (self.num_training_steps - global_step) / (max(1, self.num_training_steps - self.num_warmup_steps)) ) * self.learning_rate 4.模型训练
num_epochs = 1 warmup_steps = 2000 learning_rate = 1.5e-4 num_training_steps = num_epochs * train_dataset.get_dataset_size() from mindspore import nn from mindnlp.transformers import GPT2Config, GPT2LMHeadModel config = GPT2Config(vocab_size=len(tokenizer)) model = GPT2ForSummarization(config) lr_scheduler = LinearWithWarmUp(learning_rate=learning_rate, num_warmup_steps=warmup_steps, num_training_steps=num_training_steps) optimizer = nn.AdamWeightDecay(model.trainable_params(), learning_rate=lr_scheduler) # 记录模型参数数量 print('number of model parameters: {}'.format(model.num_parameters())) from mindnlp._legacy.engine import Trainer
from mindnlp._legacy.engine.callbacks import CheckpointCallback
ckpoint_cb = CheckpointCallback(save_path=‘checkpoint’, ckpt_name=‘gpt2_summarization’,
epochs=1, keep_checkpoint_max=2)
trainer = Trainer(network=model, train_dataset=train_dataset,
epochs=1, optimizer=optimizer, callbacks=ckpoint_cb)
trainer.set_amp(level=‘O1’) # 开启混合精度
5.、模型推理
数据处理，将向量数据变为中文数据
def process_test_dataset(dataset, tokenizer, batch_size=1, max_seq_len=1024, max_summary_len=100): def read_map(text): data = json.loads(text.tobytes()) return np.array(data['article']), np.array(data['summarization']) def pad(article): tokenized = tokenizer(text=article, truncation=True, max_length=max_seq_len-max_summary_len) return tokenized['input_ids'] dataset = dataset.map(read_map, 'text', ['article', 'summary']) dataset = dataset.map(pad, 'article', ['input_ids']) dataset = dataset.batch(batch_size) return dataset` test_dataset = process_test_dataset(test_dataset, tokenizer, batch_size=1) print(next(test_dataset.create_tuple_iterator(output_numpy=True))) model = GPT2LMHeadModel.from_pretrained('./checkpoint/gpt2_summarization_epoch_0.ckpt', config=config) model.set_train(False) model.config.eos_token_id = model.config.sep_token_id i = 0 for (input_ids, raw_summary) in test_dataset.create_tuple_iterator(): output_ids = model.generate(input_ids, max_new_tokens=50, num_beams=5, no_repeat_ngram_size=2) output_text = tokenizer.decode(output_ids[0].tolist()) print(output_text) i += 1 if i == 1: break 这节内容就学习到这里~
