小钢炮进化，MiniCPM 3.0 开源！4B参数超GPT3.5性能，无限长文本，超强RAG三件套！模型推理、微调实战来啦！

modelscope download --model=OpenBMB/MiniCPM3-4B --local_dir ./MiniCPM3-4B

from modelscope import AutoModelForCausalLM, AutoTokenizerimport torch
path = "OpenBMB/MiniCPM3-4B"device = "cuda"
tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)model = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.bfloat16, device_map=device, trust_remote_code=True)
messages = [{"role": "user", "content": "推荐5个北京的景点。"},]model_inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to(device)
model_outputs = model.generate(model_inputs,max_new_tokens=1024,top_p=0.7,temperature=0.7,repetition_penalty=1.02)
output_token_ids = [model_outputs[i][len(model_inputs[i]):] for i in range(len(model_inputs))]
responses = tokenizer.batch_decode(output_token_ids, skip_special_tokens=True)[0]print(responses)

from peft import PeftModelfrom transformers import AutoModelForCausalLM, AutoTokenizerimport torchfrom modelscope import snapshot_downloadbase_model_dir = snapshot_download("OpenBMB/MiniCPM3-4B")lora_model_dir = snapshot_download("OpenBMB/MiniCPM3-RAG-LoRA")
model = AutoModelForCausalLM.from_pretrained(base_model_dir, device_map="auto",torch_dtype=torch.bfloat16).eval()tokenizer = AutoTokenizer.from_pretrained(lora_model_dir)
model = PeftModel.from_pretrained(model, lora_model_dir)
passages_list = ["In the novel 'The Silent Watcher,' the lead character is named Alex Carter. Alex is a private detective who uncovers a series of mysterious events in a small town.","Set in a quiet town, 'The Silent Watcher' follows Alex Carter, a former police officer turned private investigator, as he unravels the town's dark secrets.","'The Silent Watcher' revolves around Alex Carter's journey as he confronts his past while solving complex cases in his hometown."]instruction = "Q: What is the name of the lead character in the novel 'The Silent Watcher'?\nA:"
passages = '\n'.join(passages_list)input_text = 'Background:\n' + passages + '\n\n' + instruction
messages = [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": input_text},]prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
outputs = model.chat(tokenizer, prompt, temperature=0.8, top_p=0.8)print(outputs[0])  # The lead character in the novel 'The Silent Watcher' is named Alex Carter.

from modelscope import AutoModel, AutoTokenizerimport torchimport torch.nn.functional as F
model_name = "OpenBMB/MiniCPM-Embedding"tokenizer = AutoTokenizer.from_pretrained(model_name)model = AutoModel.from_pretrained(model_name, trust_remote_code=True, attn_implementation="flash_attention_2", torch_dtype=torch.float16).to("cuda")model.eval()
def weighted_mean_pooling(hidden, attention_mask):attention_mask_ = attention_mask * attention_mask.cumsum(dim=1)s = torch.sum(hidden * attention_mask_.unsqueeze(-1).float(), dim=1)d = attention_mask_.sum(dim=1, keepdim=True).float()reps = s / dreturn reps
@torch.no_grad()def encode(input_texts):batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors='pt', return_attention_mask=True).to("cuda")outputs = model(**batch_dict)attention_mask = batch_dict["attention_mask"]hidden = outputs.last_hidden_state
reps = weighted_mean_pooling(hidden, attention_mask) embeddings = F.normalize(reps, p=2, dim=1).detach().cpu().numpy()return embeddings
queries = ["中国的首都是哪里？"]passages = ["beijing", "shanghai"]

INSTRUCTION = "Query: "queries = [INSTRUCTION + query for query in queries]
embeddings_query = encode(queries)embeddings_doc = encode(passages)
scores = (embeddings_query @ embeddings_doc.T)print(scores.tolist())# [[0.3535913825035095, 0.18596848845481873]]

from transformers import AutoModel, AutoTokenizer, AutoModelForSequenceClassificationimport torchimport numpy as np
#模型下载from modelscope import snapshot_downloadmodel_name = snapshot_download('OpenBMB/MiniCPM-Reranker')
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)tokenizer.padding_side = "right"model = AutoModelForSequenceClassification.from_pretrained(model_name, trust_remote_code=True,attn_implementation="flash_attention_2", torch_dtype=torch.float16).to("cuda")model.eval()max_len_q, max_len_d = 512, 512
def tokenize_our(query,doc):input_id_query = tokenizer.encode(query, add_special_tokens=False, max_length=max_len_q, truncation=True)input_id_doc = tokenizer.encode(doc, add_special_tokens=False, max_length=max_len_d, truncation=True)pad_input = {"input_ids": [tokenizer.bos_token_id] + input_id_query + [tokenizer.eos_token_id] + input_id_doc}return tokenizer.pad(pad_input,padding="max_length",max_length=max_len_q + max_len_d + 2,return_tensors="pt",)
@torch.no_grad()def rerank(input_query, input_docs):tokenized_inputs = [tokenize_our(input_query, input_doc).to("cuda") for input_doc in input_docs]input_ids = {"input_ids": [tokenized_input["input_ids"] for tokenized_input in tokenized_inputs],"attention_mask": [tokenized_input["attention_mask"] for tokenized_input in tokenized_inputs]}
for k in input_ids:input_ids[k] = torch.stack(input_ids[k]).to("cuda")outputs = model(**input_ids)score = outputs.logitsreturn score.float().detach().cpu().numpy()
queries = ["中国的首都是哪里？"]passages = [["beijing", "shanghai"]]
INSTRUCTION = "Query: "queries = [INSTRUCTION + query for query in queries]
scores = []for i in range(len(queries)):print(queries[i])scores.append(rerank(queries[i],passages[i]))
print(np.array(scores))# [[[-4.7421875][-8.8515625]]]

# 安装ms-swiftgit clone https://github.com/modelscope/ms-swift.gitcd swiftpip install -e .[llm]

# 单卡A10/3090可运行# 13GB GPU memoryCUDA_VISIBLE_DEVICES=0 swift sft \--model_type minicpm3-4b \--model_id_or_path OpenBMB/MiniCPM3-4B \--dataset jd-sentiment-zh \--learning_rate 1e-4 \--output_dir output \--lora_target_modules ALL \

# 4 * 13GB GPU memory# Deepspeed-ZeRO2NPROC_PER_NODE=4 CUDA_VISIBLE_DEVICES=0,1,2,3 \swift sft \--model_type minicpm3-4b \--model_id_or_path OpenBMB/MiniCPM3-4B \--dataset jd-sentiment-zh \--learning_rate 1e-4 \--output_dir output \--lora_target_modules ALL \--deepspeed default-zero2

CUDA_VISIBLE_DEVICES=0 swift infer \--ckpt_dir output/minicpm3-4b/vx-xxx/checkpoint-xxx \--load_dataset_config true --show_dataset_sample 10 \--do_sample false 
# merge-lora并推理CUDA_VISIBLE_DEVICES=0 swift infer \--ckpt_dir output/minicpm3-4b/vx-xxx/checkpoint-xxx \--load_dataset_config true --show_dataset_sample 10 \--merge_lora true --do-sample false