import torchfrom modelscope import AutoModelForCausalLM, AutoTokenizerfrom transformers import pipeline
torch.random.manual_seed(0)
model = AutoModelForCausalLM.from_pretrained(    "LLM-Research/Phi-3.5-mini-instruct",     device_map="cuda",     torch_dtype="auto",     trust_remote_code=True, )tokenizer = AutoTokenizer.from_pretrained("LLM-Research/Phi-3.5-mini-instruct")
messages = "<|system|>\n 你是我的人工智能助手，协助我用中文解答问题.\n<|end|><|user|>\n 你知道长沙吗？? \n<|end|><|assistant|>"
pipe = pipeline(    "text-generation",    model=model,    tokenizer=tokenizer,)
generation_args = {    "max_new_tokens": 500,    "return_full_text": False,    "temperature": 0.0,    "do_sample": False,}
output = pipe(messages, **generation_args)print(output[0]['generated_text'])

from PIL import Image import requests from transformers import AutoModelForCausalLM from transformers import AutoProcessor from modelscope import snapshot_download
model_id = snapshot_download("LLM-Research/Phi-3.5-vision-instruct")
# Note: set _attn_implementation='eager' if you don't have flash_attn installedmodel = AutoModelForCausalLM.from_pretrained(  model_id,   device_map="cuda",   trust_remote_code=True,   torch_dtype="auto",   _attn_implementation='flash_attention_2'    )
# for best performance, use num_crops=4 for multi-frame, num_crops=16 for single-frame.processor = AutoProcessor.from_pretrained(model_id,   trust_remote_code=True,   num_crops=4) 
images = []placeholder = ""
# Note: if OOM, you might consider reduce number of frames in this example.for i in range(1,20):    url = f"https://modelscope.oss-cn-beijing.aliyuncs.com/resource/Introduction-to-Microsoft-Azure-Cloud-{i}-2048.webp"     images.append(Image.open(requests.get(url, stream=True).raw))    placeholder += f"<|image_{i}|>\n"
messages = [    {"role": "user", "content": placeholder+"Summarize the deck of slides."},]
prompt = processor.tokenizer.apply_chat_template(  messages,   tokenize=False,   add_generation_prompt=True)
inputs = processor(prompt, images, return_tensors="pt").to("cuda:0") 
generation_args = {     "max_new_tokens": 1000,     "temperature": 0.0,     "do_sample": False, } 
generate_ids = model.generate(**inputs,   eos_token_id=processor.tokenizer.eos_token_id,   **generation_args)
# remove input tokens generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]response = processor.batch_decode(generate_ids,   skip_special_tokens=True,   clean_up_tokenization_spaces=False)[0] 
print(response)

import torchfrom transformers import AutoModelForCausalLM, AutoTokenizer, pipeline from modelscope import snapshot_downloadmodel_dir = snapshot_download("LLM-Research/Phi-3.5-MoE-instruct")torch.random.manual_seed(0) 
model = AutoModelForCausalLM.from_pretrained(     model_dir,      device_map="cuda",      torch_dtype="auto",      trust_remote_code=True,  ) 
tokenizer = AutoTokenizer.from_pretrained(model_dir) 

pipe = pipeline(     "text-generation",     model=model,     tokenizer=tokenizer, ) 
generation_args = {     "max_new_tokens": 500,     "return_full_text": False,     "temperature": 0.0,     "do_sample": False, } 

sys_msg = """You are a helpful AI assistant, you are an agent capable of using a variety of tools to answer a question. Here are a few of the tools available to you:
- Blog: This tool helps you describe a certain knowledge point and content, and finally write it into Twitter or Facebook style content- Translate: This is a tool that helps you translate into any language, using plain language as required
To use these tools you must always respond in JSON format containing `"tool_name"` and `"input"` key-value pairs. For example, to answer the question, "Build Muliti Agents with MOE models" you must use the calculator tool like so:
```json
{    "tool_name": "Blog",    "input": "Build Muliti Agents with MOE models"}
```
Or to translate the question "can you introduce yourself in Chinese" you must respond:
```json
{    "tool_name": "Search",    "input": "can you introduce yourself in Chinese"}
```
Remember just output the final result, ouput in JSON format containing `"agentid"`,`"tool_name"` , `"input"` and `"output"`  key-value pairs .:
```json
[

{   "agentid": "step1",    "tool_name": "Blog",    "input": "Build Muliti Agents with MOE models",    "output": "........."},
{   "agentid": "step2",    "tool_name": "Search",    "input": "can you introduce yourself in Chinese",    "output": "........."},{    "agentid": "final"    "tool_name": "Result,    "output": "........."}]
```
The users answer is as follows."""

def instruction_format(sys_message: str, query: str):    # note, don't "</s>" to the end    return f'<|system|> {sys_message} <|end|>\n<|user|> {query} <|end|>\n<|assistant|>'

query ='Write something about Generative AI with MOE , translate it to Chinese'input_prompt = instruction_format(sys_msg, query)
import os
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True "output = pipe(input_prompt, **generation_args)output[0]['generated_text']

modelscope download --model=LLM-Research/Phi-3.5-mini-instruct-GGUF --local_dir . Phi-3.5-mini-instruct-Q5_K_M.gguf

modelscope download --model=modelscope/ollama-linux --local_dir ./ollama-linuxcd ollama-linuxsudo chmod 777 ./ollama-modelscope-install.sh./ollama-modelscope-install.sh

ollama serve

FROM /mnt/workspace/Phi-3.5-mini-instruct-Q5_K_M.ggufTEMPLATE """{{ if .System }}<|system|>
{{ .System }}<|end|>
{{ end }}{{ if .Prompt }}<|user|>
{{ .Prompt }}<|end|>
{{ end }}<|assistant|>
{{ .Response }}<|end|>"""

ollama create myphi3_5 --file ./Modelfile

ollama run myphi3_5

git clone https://github.com/modelscope/swift.gitcd swiftpip install -e .[llm]
# 可选, 对phi3_5-mini-instruct进行推理加速.pip install vllm

{"query": "<image>55555", "response": "66666", "images": ["image_path"]}{"query": "eeeee<image>eeeee<image>eeeee", "response": "fffff", "history": [], "images": ["image_path1", "image_path2"]}{"query": "EEEEE", "response": "FFFFF", "history": [["query1", "response2"], ["query2", "response2"]], "images": []}

# 显存占用: 4 * 12GB# 默认会将lora_target_modules设置为llm和projector所有的linearCUDA_VISIBLE_DEVICES=0,1,2,3 NPROC_PER_NODE=4 swift sft \  --model_type phi3_5-vision-instruct \  --model_id_or_path LLM-Research/Phi-3.5-vision-instruct \  --sft_type lora \  --dataset coco-en-mini#20000 \  --deepspeed default-zero2

  --dataset train.jsonl \  --val_dataset val.jsonl \

# 推理CUDA_VISIBLE_DEVICES=0 swift infer \    --ckpt_dir output/phi3_5-vision-instruct/vx-xxx/checkpoint-xxx \    --load_dataset_config true
# merge-lora并推理CUDA_VISIBLE_DEVICES=0 swift infer \    --ckpt_dir output/phi3_5-vision-instruct/vx-xxx/checkpoint-xxx \    --load_dataset_config true --merge_lora true \    --safe_serialization false