from modelscope import AutoModelForCausalLM, AutoTokenizer
model_name = "qwen/Qwen2.5-7B-Instruct"
model = AutoModelForCausalLM.from_pretrained(model_name,torch_dtype="auto",device_map="auto")tokenizer = AutoTokenizer.from_pretrained(model_name)
prompt = "Give me a short introduction to large language model."messages = [{"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."},{"role": "user", "content": prompt}]text = tokenizer.apply_chat_template(messages,tokenize=False,add_generation_prompt=True)model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
generated_ids = model.generate(**model_inputs,max_new_tokens=512)generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]

# 安装ms-swiftgit clone https://github.com/modelscope/ms-swift.gitcd ms-swiftpip install -e .[llm]
# qwen2-vl# https://github.com/QwenLM/Qwen2-VL/issues/96pip install git+https://github.com/huggingface/transformers@21fac7abba2a37fae86106f87fcf9974fd1e3830# vllm加速pip install vllm>=0.6.1

# 实验环境：4 * A100# 显存占用：4 * 70GBNPROC_PER_NODE=4 CUDA_VISIBLE_DEVICES=0,1,2,3 swift sft \--model_type qwen2_5-72b-instruct \--model_id_or_path qwen/Qwen2.5-72B-Instruct \--dataset qwen2-pro-en#500 qwen2-pro-zh#500 self-cognition#500 \--logging_steps 5 \--learning_rate 1e-4 \--output_dir output \--lora_target_modules ALL \--model_name 小黄 'Xiao Huang' \--model_author 魔搭 ModelScope \--deepspeed default-zero3
# 单卡A10/3090可运行的例子 （Qwen2.5-7B-Instruct）# 显存占用：24GBCUDA_VISIBLE_DEVICES=0 swift sft \--model_type qwen2_5-7b-instruct \--model_id_or_path qwen/Qwen2.5-7B-Instruct \--dataset qwen2-pro-en#500 qwen2-pro-zh#500 self-cognition#500 \--logging_steps 5 \--max_length 2048 \--learning_rate 1e-4 \--output_dir output \--lora_target_modules ALL \--model_name 小黄 'Xiao Huang' \--model_author 魔搭 ModelScope

# 直接推理CUDA_VISIBLE_DEVICES=0,1 swift infer \--ckpt_dir output/qwen2_5-72b-instruct/vx-xxx/checkpoint-xxx \
# merge-lora并使用vLLM进行推理加速CUDA_VISIBLE_DEVICES=0,1 swift export \--ckpt_dir output/qwen2_5-72b-instruct/vx-xxx/checkpoint-xxx \--merge_lora true
CUDA_VISIBLE_DEVICES=0,1,2,3 swift infer \--ckpt_dir output/qwen2_5-72b-instruct/vx-xxx/checkpoint-xxx-merged \--infer_backend vllm --max_model_len 8192 \    --tensor_parallel_size 4

# 实验环境：8 * A100# 显存占用：SIZE_FACTOR=8 MAX_PIXELS=602112 \NPROC_PER_NODE=8 CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \swift sft \--model_type qwen2-vl-72b-instruct \--model_id_or_path qwen/Qwen2-VL-72B-Instruct \--sft_type lora \--dataset latex-ocr-print#20000 \--deepspeed default-zero3如果要使用自定义数据集，只需按以下方式进行指定：# val_dataset可选，如果不指定，则会从dataset中切出一部分数据集作为验证集--dataset train.jsonl \--val_dataset val.jsonl \自定义数据集格式：{"query": "<image>55555", "response": "66666", "images": ["image_path"]}{"query": "<image><image>eeeee", "response": "fffff", "history": [], "audios": ["image_path1", "image_path2"]}{"query": "EEEEE", "response": "FFFFF", "history": [["query1", "response1"], ["query2", "response2"]]}

# 直接推理CUDA_VISIBLE_DEVICES=0,1 swift infer \--ckpt_dir output/qwen2-vl-72b-instruct/vx-xxx/checkpoint-xxx \--load_dataset_config true
# merge-lora并使用vLLM进行推理加速CUDA_VISIBLE_DEVICES=0,1 swift export \--ckpt_dir output/qwen2-vl-72b-instruct/vx-xxx/checkpoint-xxx \--merge_lora true
CUDA_VISIBLE_DEVICES=0,1,2,3 swift infer \--ckpt_dir output/qwen2-vl-72b-instruct/vx-xxx/checkpoint-xxx-merged \--load_dataset_config true --infer_backend vllm \--tensor_parallel_size 4 --max_model_len 16384

modelscope download --model=qwen/Qwen2.5-7B-Instruct --local_dir ./Qwen2.5-7B-Instructvllm serve ./Qwen2.5-7B-Instruct --enable-auto-tool-choice --tool-call-parser hermes

modelscope download --model=qwen/Qwen2.5-3B-Instruct-GGUF--local_dir ./ qwen2.5-3b-instruct-q5_k_m.ggufollama create qwen2_5 -f /mnt/workspace/Modelfileollama run qwen2_5

FROM /mnt/workspace/qwen2.5-3b-instruct-q5_k_m.gguf
# set the temperature to 0.7 [higher is more creative, lower is more coherent]PARAMETER temperature 0.7PARAMETER top_p 0.8PARAMETER repeat_penalty 1.05TEMPLATE """{{ if .Messages }}{{- if or .System .Tools }}<|im_start|>system{{ .System }}{{- if .Tools }}
# Tools
You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools:<tools>{{- range .Tools }}{{ .Function }}{{- end }}</tools>
For each function call, return a JSON object with function name and arguments within <tool_call></tool_call> XML tags as follows:<tool_call>{"name": <function-name>, "arguments": <args-json-object>}</tool_call>{{- end }}<|im_end|>{{- end }}{{- range .Messages }}{{- if eq .Role "user" }}<|im_start|>{{ .Role }}{{ .Content }}<|im_end|>{{- else if eq .Role "assistant" }}<|im_start|>{{ .Role }}{{- if .Content }}{{ .Content }}{{- end }}{{- if .ToolCalls }}<tool_call>{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}}{{ end }}</tool_call>{{- end }}<|im_end|>{{- else if eq .Role "tool" }}<|im_start|>user<tool_response>{{ .Content }}</tool_response><|im_end|>{{- end }}{{- end }}<|im_start|>assistant{{ else }}{{ if .System }}<|im_start|>system{{ .System }}<|im_end|>{{ end }}{{ if .Prompt }}<|im_start|>user{{ .Prompt }}<|im_end|>{{ end }}<|im_start|>assistant{{ end }}"""