大模型的监督微调

1.Chat Template

1. '1. 指令微调在预训练（LoRA微调）之后'
2. # 预训练使大模型成为“领域专家”
3. # 指令微调令大模型学会表达
4. 每一个大模型的指令微调都不一样；
5. 所以一定要根据官方发布的格式做指令微调

复制代码

1. '2. Chat_Template的源代码'
2. # Chat_Template.py
3. from transformers import AutoModelForCausalLM, AutoTokenizer
4. import torch
6. model_path = r'D:\work\models\Meta-Llama-3.1-8B-Instruct'
8. tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
9. model = AutoModelForCausalLM.from_pretrained(model_path).to("cuda")
10. optimizer = torch.optim.AdamW(model.parameters())
12. dialog = [{"role": "system", "content": "You are a helpful assistant."},
13.           {"role": "user", "content": "天空为什么是蓝色的？"},
14.           {"role": "assistant", "content": "这是由于光的散射引起的。"}]
16. input = tokenizer.apply_chat_template(dialog, return_tensors="pt")
17. input = {k: v.to("cuda") for k, v in input.items()}
19. #设置labels和inputs一致
20. input["labels"] = input["input_ids"].clone()
22. output = model(**input)
24. #获取模型的loss
25. loss = output.loss
26. loss.backward()
27. optimizer.step()
28. optimizer.zero_grad()
30. #保存模型
31. model.save_pretrained("output_dir")

复制代码

2.Completions only

1. '1. 只对回答部分做计算loss'
2. 由于Chat Template会对指令的所有内容计算loss
3. 为了优化该部分，使用Completions only

复制代码

1. '2. Completions onlyの源代码'
2. # Completions_only.py
3. import functools
4. import json
5. from peft import LoraConfig, TaskType, get_peft_model
6. from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
7. from torch.utils.data import DataLoader, Dataset
8. import torch
11. class SFTDataset(Dataset):
13.     def __init__(self, file_path, tokenizer):
14.         super().__init__()
15.         self.file_path = file_path
16.         self.examples = self._load_data(self.file_path)
17.         self.tokenizer = tokenizer
19.     @staticmethod
20.     def _load_data(file_path):
21.         items = []
22.         with open(file_path, "r", encoding="utf8")as f:
23.             for line in f:
24.                 item = json.loads(line)
25.                 items.append(item)
26.         return items
28.     def __getitem__(self, index):
29.         example = self.examples[index]
30.         dialog = [{"role": "system", "content": "You are a helpful assistant."},
31.                   {"role": "user", "content": example["query"]},
32.                   {"role": "assistant", "content": example["answer"]}]
33.         chat = tokenizer.apply_chat_template(dialog, tokenize=False)
34.         return chat
36.     def __len__(self):
37.         return len(self.examples)
40. model_path = r'D:\work\models\Meta-Llama-3.1-8B-Instruct'
41. tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
42. tokenizer.padding_side = "right"
43. tokenizer.pad_token = tokenizer.eos_token
44. bnb_config = BitsAndBytesConfig(
45.     load_in_4bit=True,
46.     bnb_4bit_use_double_quant=True,
47.     bnb_4bit_quant_type="nf4",
48.     bnb_4bit_compute_dtype=torch.bfloat16
49. )
50. model = AutoModelForCausalLM.from_pretrained(model_path, quantization_config=bnb_config)
51. peft_config = LoraConfig(
52.     r=8,
53.     target_modules=["q_proj",
54.                     "v_proj",
55.                     "k_proj",
56.                     "o_proj",
57.                     "gate_proj",
58.                     "down_proj",
59.                     "up_proj"
60.                     ],
61.     task_type=TaskType.CAUSAL_LM,
62.     lora_alpha=16,
63.     lora_dropout=0.05
64. )
65. model = get_peft_model(model, peft_config)
66. model.print_trainable_parameters()
67. model.to("cuda")
69. optimizer = torch.optim.AdamW(model.parameters())
72. def sft_collate(batch, tokenizer, end_str, max_length):
73.     end_str = "<|start_header_id|>assistant<|end_header_id""|>\n\n"
74.     inputs = tokenizer(batch, max_length=max_length, padding=True, truncation=True)
75.     input_ids = inputs["input_ids"]
76.     input_len = len(input_ids[0])
77.     end_ids = tokenizer(end_str)["input_ids"]
78.     end_id_len = len(end_ids)
79.     loss_mask = []
80.     for input_id in input_ids:
81.         for i in range(len(input_id) - end_id_len, -1, -1):
82.             if input_id[i:i + end_id_len] == end_ids:
83.                 mask = [1] * (input_len - 1)
84.                 mask[:i + end_id_len - 1] = [0] * (i + end_id_len - 1)
85.                 loss_mask.append(mask)
86.                 break
87.             if i == 0:  # 所有回答部分都被截断
88.                 loss_mask.append([0] * (input_len - 1))
89.     inputs = {k: torch.tensor(v) for k, v in inputs.items()}
90.     loss_mask = torch.tensor(loss_mask)
91.     return inputs, loss_mask
94. collate_fn = functools.partial(sft_collate,
95.                                tokenizer=tokenizer,
96.                                end_str="<|start_header_id|>assistant<|end_header_id""|>\n\n",
97.                                max_length=50)
99. sft_dataset = SFTDataset("./data/sft_data.json", tokenizer)
100. data_loader = DataLoader(sft_dataset, batch_size=2, collate_fn=collate_fn, shuffle=True)
101. epoch = 10
103. for i in range(epoch):
104.     for inputs, loss_mask in data_loader:
105.         inputs = {k: v.to("cuda") for k, v in inputs.items()}
106.         loss_mask = loss_mask.to("cuda")
107.         logits = model(**inputs).logits[:, :-1, :]
108.         labels = inputs["input_ids"][:, 1:]
109.         logits = logits.reshape(-1, logits.size(-1))
110.         labels = labels.reshape(-1)
111.         loss_mask = loss_mask.reshape(-1)
113.         loss = torch.nn.functional.cross_entropy(logits, labels, reduction="none")
114.         loss = loss * loss_mask
115.         loss = torch.mean(loss)
116.         loss.backward()
117.         optimizer.step()
118.         optimizer.zero_grad()
119.     print(loss.item())

复制代码

3.NEFTune

1. '1. Noisy Embeddings Finetuning'
2. 通过embedding，将离散对象映射到连续向量空间中的点
3. 对embedding做噪声处理，可以提高模型的表现

复制代码

1. '2. NEFTuneの源代码'
3. import functools
4. import json
5. from peft import LoraConfig, TaskType, get_peft_model
6. from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
7. from torch.utils.data import DataLoader, Dataset
8. import torch
11. class SFTDataset(Dataset):
13.     def __init__(self, file_path, tokenizer):
14.         super().__init__()
15.         self.file_path = file_path
16.         self.examples = self._load_data(self.file_path)
17.         self.tokenizer = tokenizer
19.     @staticmethod
20.     def _load_data(file_path):
21.         items = []
22.         with open(file_path, "r", encoding="utf8")as f:
23.             for line in f:
24.                 item = json.loads(line)
25.                 items.append(item)
26.         return items
28.     def __getitem__(self, index):
29.         example = self.examples[index]
30.         dialog = [{"role": "system", "content": "You are a helpful assistant."},
31.                   {"role": "user", "content": example["query"]},
32.                   {"role": "assistant", "content": example["answer"]}]
33.         chat = tokenizer.apply_chat_template(dialog, tokenize=False)
34.         return chat
36.     def __len__(self):
37.         return len(self.examples)
40. model_path = r'D:\work\models\Meta-Llama-3.1-8B-Instruct'
41. tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
42. tokenizer.padding_side = "right"
43. tokenizer.pad_token = tokenizer.eos_token
44. bnb_config = BitsAndBytesConfig(
45.     load_in_4bit=True,
46.     bnb_4bit_use_double_quant=True,
47.     bnb_4bit_quant_type="nf4",
48.     bnb_4bit_compute_dtype=torch.bfloat16
49. )
50. model = AutoModelForCausalLM.from_pretrained(model_path, quantization_config=bnb_config)
51. peft_config = LoraConfig(
52.     r=8,
53.     target_modules=["q_proj",
54.                     "v_proj",
55.                     "k_proj",
56.                     "o_proj",
57.                     "gate_proj",
58.                     "down_proj",
59.                     "up_proj"
60.                     ],
61.     task_type=TaskType.CAUSAL_LM,
62.     lora_alpha=16,
63.     lora_dropout=0.05
64. )
65. model = get_peft_model(model, peft_config)
66. model.print_trainable_parameters()
67. model.to("cuda")
69. optimizer = torch.optim.AdamW(model.parameters())
72. def sft_collate(batch, tokenizer, end_str, max_length):
73.     inputs = tokenizer(batch, max_length=max_length, padding=True, truncation=True)
74.     input_ids = inputs["input_ids"]
75.     input_len = len(input_ids[0])
76.     end_ids = tokenizer(end_str)["input_ids"]
77.     end_id_len = len(end_ids)
78.     loss_mask = []
79.     for input_id in input_ids:
80.         for i in range(len(input_id) - end_id_len, -1, -1):
81.             if input_id[i:i + end_id_len] == end_ids:
82.                 mask = [1] * (input_len - 1)
83.                 mask[:i + end_id_len - 1] = [0] * (i + end_id_len - 1)
84.                 loss_mask.append(mask)
85.                 break
86.             if i == 0:  # 所有回答部分都被截断
87.                 loss_mask.append([0] * (input_len - 1))
88.     inputs = {k: torch.tensor(v) for k, v in inputs.items()}
89.     loss_mask = torch.tensor(loss_mask)
90.     return inputs, loss_mask
93. collate_fn = functools.partial(sft_collate,
94.                                tokenizer=tokenizer,
95.                                end_str="<|start_header_id|>assistant<|end_header_id""|>\n\n",
96.                                max_length=500)
98. sft_dataset = SFTDataset("./data/sft_data.json", tokenizer)
99. data_loader = DataLoader(sft_dataset, batch_size=2, collate_fn=collate_fn, shuffle=True)
100. epoch = 10
101. neftune_noise_alpha = 10
102. for i in range(epoch):
103.     for inputs, loss_mask in data_loader:
104.         input_ids = inputs.pop("input_ids")
105.         input_embeddings = model.base_model.model.model.embed_tokens(input_ids)
106.         dims = torch.tensor(input_embeddings.size(1) * input_embeddings.size(2))
107.         mag_norm = neftune_noise_alpha / torch.sqrt(dims)
108.         input_embeddings = input_embeddings + torch.zeros_like(input_embeddings).uniform_(-mag_norm, mag_norm)
109.         inputs["inputs_embeds"] = input_embeddings
110.         inputs = {k: v.to("cuda") for k, v in inputs.items()}
111.         loss_mask = loss_mask.to("cuda")
112.         logits = model(**inputs).logits[:, :-1, :]
113.         labels = input_ids[:, 1:].to("cuda")
114.         logits = logits.reshape(-1, logits.size(-1))
115.         labels = labels.reshape(-1)
116.         loss_mask = loss_mask.reshape(-1)
118.         loss = torch.nn.functional.cross_entropy(logits, labels, reduction="none")
119.         loss = loss * loss_mask
120.         loss = torch.mean(loss)
121.         loss.backward()
122.         optimizer.step()
123.         optimizer.zero_grad()
124.     print(loss.item())

复制代码

4.SFT_Trainer

1. # 用Trainer包监督微调大模型
2. import json
4. import torch
5. from datasets import Dataset
6. from peft import LoraConfig, get_peft_model, TaskType
7. from transformers import AutoTokenizer, BitsAndBytesConfig, AutoModelForCausalLM
8. from trl import SFTTrainer, SFTConfig, DataCollatorForCompletionOnlyLM
10. items = []
11. with open("./data/sft_data.json", "r", encoding="utf8")as f:
12.     for line in f:
13.         item = json.loads(line)
14.         items.append({"prompt": item["query"], "completion": item["answer"]})
15. dataset = Dataset.from_list(items)
17. model_path = r'D:\work\models\Meta-Llama-3.1-8B-Instruct'
18. tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
19. tokenizer.padding_side = "right"
20. tokenizer.pad_token = tokenizer.eos_token
21. bnb_config = BitsAndBytesConfig(
22.     load_in_4bit=True,
23.     bnb_4bit_use_double_quant=True,
24.     bnb_4bit_quant_type="nf4",
25.     bnb_4bit_compute_dtype=torch.float16
26. )
27. model = AutoModelForCausalLM.from_pretrained(model_path, quantization_config=bnb_config,torch_dtype=torch.float16)
28. peft_config = LoraConfig(
29.     r=8,
30.     target_modules=["q_proj",
31.                     "v_proj",
32.                     "k_proj",
33.                     "o_proj",
34.                     "gate_proj",
35.                     "down_proj",
36.                     "up_proj"
37.                     ],
38.     task_type=TaskType.CAUSAL_LM,
39.     lora_alpha=16,
40.     lora_dropout=0.05
41. )
42. model = get_peft_model(model, peft_config)
43. sft_config = SFTConfig(output_dir="/tmp",
44.                        neftune_noise_alpha=10,
45.                        per_device_train_batch_size=1,
46.                        max_seq_length=100,
47.                        num_train_epochs=10,
48.                        logging_steps=10,
49.                        logging_strategy="steps")
50. response_template = "<|start_header_id|>assistant<|end_header_id""|>\n\n"
51. collator = DataCollatorForCompletionOnlyLM(response_template, tokenizer=tokenizer)
52. trainer = SFTTrainer(
53.     model=model,
54.     train_dataset=dataset,
55.     args=sft_config,
56.     data_collator=collator
58. )
60. trainer.train()

复制代码

学习视频：【大模型微调看这个视频就够了 SFT NEFTune】 https://www.bilibili.com/video/BV1gmWDeLEMZ/?share_source=copy_web&vd_source=050ab764db52d186ab224170392c4055

来源：豆瓜网用户自行投稿发布，如果侵权，请联系站长删除