from transformers import pipeline
import torch

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
output = pipe(prompt, max_length=100, do_sample=False)
print(output[0]["generated_text"])

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. トークナイザとモデルのロード
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16).eval()

# 3. プロンプト（入力文）
prompt = "Tell me a story about a dragon."

# 4. トークナイズ処理
inputs = tokenizer(prompt, return_tensors="pt")
## 実際には以下の処理を一気に行っている。
## token = tokenizer.tokenize(prompt)
## input_ids = tokenizer.convert_tokens_to_ids(tokens)
## input_ids_tensor = torch.tensor([input_ids])
## attention_mask = torch.ones_like(input_ids_tensor)
##
## inputs = {
##     "input_ids": input_ids_tensor,
##     "attention_mask": attention_mask,
## }

# 5. 生成
output_ids = model.generate(
  **inputs,
  max_length=100,
  do_sample=False,
)

# 6. トークン列をテキストに変換
generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)
print(generated_text)
## tokens = tokenizer.convert_ids_to_tokens(output_ids[0])
## text = tokenizer.convert_tokens_to_string(tokens)

from transformers import pipeline
import torch

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
output = pipe(prompt, max_length=100, do_sample=True) # 変更箇所
print(output[0]["generated_text"])

...
# 4. ここでプロンプトをLLMに入力して、出力を得る。（注意：output-> outputsに変数名変更している）
outputs = pipe(prompt, max_length=100, do_sample=True, num_return_sequences=3) # 変更箇所
print(outputs)
# 以下出力結果
[
  {'generated_text': 'Tell me a story about a dragon.\n\nI want to hear a story about a dragon that is a little bit different from the usual ones. Can you tell me a story about a dragon who is not evil, but rather a protector of the land and its people?\n\nOf course! Here is a story about a dragon who is not evil, but rather a protector of the land and its people:\n\nOnce upon a time, in a far'}, 
  {'generated_text': 'Tell me a story about a dragon.\n\nOnce upon a time, in a far-off land, there was a magnificent dragon named Scorch. Scorch was unlike any dragon that had ever been seen before. He was covered in shimmering scales of gold and silver, and his wings were as wide as the horizon. He had a fiery mane that flowed like a river of flame, and his eyes glowed like embers from the'}, 
  {'generated_text': 'Tell me a story about a dragon.\n\nDragon Story\n\nOnce upon a time, in a land far, far away, there lived a magnificent dragon named Scorch. Scorch was unlike any other dragon in the land, for he had a heart of gold and a fierce determination to protect his home and the creatures that lived within it.\n\nScorch lived in a vast, sprawling cave system deep within a great mountain range.'}
]

from transformers import pipeline
import torch
import sacrebleu # 定量的に多様性を測るため、metricを追加

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
outputs = pipe(prompt, max_length=100, do_sample=True, num_return_sequences=3,
              top_p=0.9, temperature=0.8) # 追記箇所
print(outputs)

# (Option) Self-BLEUの計算。低ければ低いほうがいい。
scores = []
generated_texts = [output["generated_text"].strip() for output in outputs]
for i, hypo in enumerate(generated_texts):
    refs = generated_texts[:i] + generated_texts[i+1:] # 他の全てを参照とする
    bleu = sacrebleu.corpus_bleu([hypo], [refs]) # sacrebleuはリストを受け取る
    scores.append(bleu.score)

# Self-BLEUの結果。低ければ低いほうが、多様性が高いので注意。
for i, score in enumerate(scores):
    print(f"Self-BLEU for output {i+1}: {score:.2f}")
print(f"Average Self-BLEU: {sum(scores)/len(scores):.2f}")

[
  {'generated_text': 'Tell me a story about a dragon.\n\n---\n\nOnce upon a time, in a land far, far away, there lived a magnificent dragon named Scorch. Scorch was unlike any dragon that had ever been seen before. He was covered in shimmering scales that glistened in the sunlight, and his wings stretched out wide enough to block out the entire sky.\n\nScorch lived in a great, sprawling cave deep within'},
  {'generated_text': 'Tell me a story about a dragon.\n\n"The dragon\'s name was Scorch, and he was the most fearsome creature in all the land. He had scales as black as coal, wings as wide as a castle, and a fiery breath that could melt steel. Scorch lived in a great cave deep within a mountain, and he spent his days sleeping and breathing fire.\n\nOne day, a brave knight named Sir Edward decided to'},
  {'generated_text': "Tell me a story about a dragon.\nI'd be happy to! Here is a story about a dragon:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was the largest and most powerful dragon in all the land, with scales as black as night and wings that shone like the brightest stars. He was a fierce creature, feared by all who lived within his domain"}
]
Self-BLEU for output 1: 19.34
Self-BLEU for output 2: 19.36
Self-BLEU for output 3: 31.67
Average Self-BLEU: 23.46

[
  {'generated_text': 'Tell me a story about a dragon.\n\nSure! Here is a story about a dragon:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was the largest and most powerful dragon in all the land, with scales as red as fire and wings that stretched as wide as the sky. He lived in a great, sprawling cave deep within a mountain, surrounded by piles of'}, 
  {'generated_text': 'Tell me a story about a dragon.\n\nSure! Here is a story about a dragon:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was the largest and most powerful dragon in all the land, with scales as red as fire and wings that stretched as wide as the sky. He lived in a great, sprawling cave deep within a mountain, surrounded by piles of'}, 
  {'generated_text': 'Tell me a story about a dragon.\n\nSure! Here is a story about a dragon:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was the largest and most powerful dragon in all the land, with scales as red as fire and wings that stretched as wide as the sky. He lived in a great, sprawling cave deep within a mountain, surrounded by piles of'}
]
Self-BLEU for output 1: 100.00
Self-BLEU for output 2: 100.00
Self-BLEU for output 3: 100.00
Average Self-BLEU: 100.00

[
  {'generated_text': 'Tell me a story about a dragon.\n\nDragon Story\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was unlike any other dragon in the land, for he had a heart of gold and a fierce determination to protect his home and the creatures within it.\n\nScorch lived in a vast, sprawling cave system deep within a mountain range. The cave was filled'},
  {'generated_text': 'Tell me a story about a dragon.\n\nSure! Here is a story about a dragon:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was the largest and most powerful dragon in all the land, with scales as black as coal and eyes that glowed like embers. He lived in a great, sprawling cave system deep within a mountain range, surrounded by piles'}, 
  {'generated_text': 'Tell me a story about a dragon.\n\nSure! Here is a story about a dragon:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was the largest and most powerful dragon in all the land, with scales as red as fire and wings that stretched wider than the horizon. He lived in a great cave high up in the mountains, surrounded by a hoard of treasure'}
]
Self-BLEU for output 1: 46.92
Self-BLEU for output 2: 46.91
Self-BLEU for output 3: 37.63
Average Self-BLEU: 43.82

[
  {'generated_text': 'Tell me a story about a dragon.\n\nDragon stories are a classic tale that has been told and retold for centuries. They are filled with magic, adventure, and of course, dragons. Here is a story about a dragon that I hope you will enjoy:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was unlike any dragon that had ever been seen before. He'},
  {'generated_text': 'Tell me a story about a dragon.\n\nSure! Here is a story about a dragon:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was the largest and most powerful dragon in all the land, with scales as red as fire and wings that stretched as wide as the sky. He lived in a great, sprawling cave deep within a mountain, surrounded by piles of'}, 
  {'generated_text': 'Tell me a story about a dragon.\n\nDragon stories are a classic tale that has been told and retold for centuries. They are filled with magic, adventure, and of course, dragons. Here is a story about a dragon that I hope you will enjoy:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was unlike any dragon that had ever been seen before. He'}
]
Self-BLEU for output 1: 42.13
Self-BLEU for output 2: 42.13
Self-BLEU for output 3: 100.00
Average Self-BLEU: 61.42

[
  {'generated_text': "Tell me a story about a dragon. I want to be amazed.\nA few days ago, a man named Tom was wandering through a dense forest when he stumbled upon a magnificent dragon. The dragon was unlike any Tom had ever seen before, with scales that shimmered in the sunlight like diamonds and wings that stretched as wide as the trees themselves.\nTom was amazed by the dragon's size and beauty, but he was"},
  {'generated_text': "Tell me a story about a dragon. 🐉\n\nSure, here's a story about a dragon:\n\nOnce upon a time, in a far-off land, there lived a magnificent dragon named Scorch. Scorch was the largest and most powerful dragon in all the land, with scales as black as coal and eyes that glowed like embers. He lived in a great cave deep within a towering mountain, surrounded"},
  {'generated_text': 'Tell me a story about a dragon.\nAsked by: Sophie M\nOnce upon a time, in a far-off land, there was a magnificent dragon named Scorch. Scorch was unlike any dragon that had ever been seen before, for he was covered in shimmering scales of gold and silver, and his wings were as wide as the sun.\n\nScorch lived in a great, sprawling cave system deep within a towering'}
]
Self-BLEU for output 1: 14.11
Self-BLEU for output 2: 14.09
Self-BLEU for output 3: 19.06
Average Self-BLEU: 15.75

[
  {'generated_text': 'Tell me a story about a dragon. This... dragone doesn�� like its home because itв�eton; a cage in his chom where his homeвъсреа where, like other creature with out feet the and wasвore but. вThis must the a small dank is. All is in pitch yoke y� dragonly see th dragО him through window bars hd th, dark gвt that has t in their p of c he'},
  {'generated_text': 'Tell me a story about a dragon. It had a... 5 Anunciados (Dragnelli # Drag) are dragoni? (Answerer... As much happiness and comfort asto my mother... Tame, sweet-facedyouth... Layw from Manaos, his mother having tchenlse. As Much happ... He t... This story doesn１ Dragon born to an old wicked wo... 5Anunciao da Morrã'}, 
  {'generated_text': 'Tell me a story about a dragon. This should not feel forced?’\nYou say ‘Ok Sure. How about the story on dragoni?” Dragone smiled broad the smile'}
]
Self-BLEU for output 1: 11.35
Self-BLEU for output 2: 11.36
Self-BLEU for output 3: 6.67
Average Self-BLEU: 9.80

# Tell me a story about a dragon.が与えられたときの、次の単語の確率と想定
[
    ("the", 0.25),
    ("a", 0.20),
    ("dragon", 0.15),
    ("flew", 0.10),
    ("over", 0.08),
    ("and", 0.07),
    ("castle", 0.06),
    ("into", 0.04),
    ("flames", 0.03),
    ("darkness", 0.02),
]

# 元の分布（T=1.0）
[
    ("the", 0.25),
    ("a", 0.20),
    ("dragon", 0.15),
    ("flew", 0.10),
    ("over", 0.08),
    ("and", 0.07),
    ("castle", 0.06),
    ("into", 0.04),
    ("flames", 0.03),
    ("darkness", 0.02),
]
# top_p=0.8のとき、"and"まで考慮される。

# T=0.5のときの分布
[
    ("the", 0.409),
    ("a", 0.262),
    ("dragon", 0.147),
    ("flew", 0.07),
    ("over", 0.04),
    ("and", 0.03),
    ("castle", 0.02),
    ("into", 0.01),
    ("flames", 0.01),
    ("darkness", 0.003),
]
# top_p=0.8のとき、"dragon"まで考慮される。

# T=1.5のときの分布
[
    ("the", 0.195),
    ("a", 0.168),
    ("dragon", 0.139),
    ("flew", 0.106),
    ("over", 0.09),
    ("and", 0.08),
    ("castle", 0.08),
    ("into", 0.06),
    ("flames", 0.05),
    ("darkness", 0.04),
]
# top_p=0.8のとき、"castle"まで考慮される。

# T=1０のときの分布
[
    ("the", 0.112),
    ("a", 0.11),
    ("dragon", 0.107),
    ("flew", 0.103),
    ("over", 0.1),
    ("and", 0.1),
    ("castle", 0.1),
    ("into", 0.1),
    ("flames", 0.1),
    ("darkness", 0.09),
]
# top_p=0.8のとき、"into"まで考慮される。

import math

q = [("the", 0.25), ("a", 0.20), ("dragon", 0.15), ("flew", 0.10), ("over", 0.08), 
     ("and", 0.07), ("castle", 0.06), ("into", 0.04), ("flames", 0.03), ("darkness", 0.02)]
T = 1.5 # 一旦1.5に設定。

exp_q = [0]*len(q) # 一旦分子を全部計算する。
p = [None]*len(q) # pを格納するリストをあらかじめ作っておく

# ここで分子を一気に計算
for i in range(len(q)):
  exp_q[i] = math.exp(math.log(q[i][1]) / T) # [1]なのはタプルだから、確率のみ取得したい

# exp_qをsumで割って、pを求める。
for i in range(len(q)):
  p[i] = (q[i][0], exp_q[i] /sum(exp_q))
print(p)

# [('the', 0.1952), ('a', 0.1683), ('dragon', 0.1389), ('flew', 0.1060), ('over', 0.0913),
#  ('and', 0.0836), ('castle', 0.0754), ('into', 0.0575), ('flames', 0.0475), ('darkness', 0.0363)]

from transformers import pipeline
import torch

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
output = pipe(prompt, max_length=100, do_sample=False)
print(output[0]["generated_text"])

...
# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)
...

from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。quantization_configが新たに追加されている。load_in_8bit=Trueとすればよい。
qconfig = BitsAndBytesConfig(load_in_8bit=True) # load_in_4bit=Trueなら4bit
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=qconfig)
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
output = pipe(prompt, max_length=100, do_sample=False)
print(output[0]["generated_text"])

import torch

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。      
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32)
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
output = pipe(prompt, max_length=100, do_sample=False)
# print(output[0]["generated_text"])

print(torch.cuda.memory_summary()) # ここでメモリ使用量を確認する

---OUTPUT---
|===========================================================================|
|                  PyTorch CUDA memory summary, device ID 0                 |
|---------------------------------------------------------------------------|
|            CUDA OOMs: 0            |        cudaMalloc retries: 0         |
|===========================================================================|
|        Metric         | Cur Usage  | Peak Usage | Tot Alloc  | Tot Freed  |
|---------------------------------------------------------------------------|
| Allocated memory      |  25713 MiB |  25815 MiB |  32526 MiB |   6813 MiB |
|       from large pool |  25712 MiB |  25814 MiB |  28776 MiB |   3064 MiB |
|       from small pool |      1 MiB |     66 MiB |   3750 MiB |   3749 MiB |
|---------------------------------------------------------------------------|
...
...

from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。      
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
output = pipe(prompt, max_length=100, do_sample=False)
# print(output[0]["generated_text"])

print(torch.cuda.memory_summary()) # ここでメモリ使用量を確認する

---OUTPUT---
|===========================================================================|
|                  PyTorch CUDA memory summary, device ID 0                 |
|---------------------------------------------------------------------------|
|            CUDA OOMs: 0            |        cudaMalloc retries: 0         |
|===========================================================================|
|        Metric         | Cur Usage  | Peak Usage | Tot Alloc  | Tot Freed  |
|---------------------------------------------------------------------------|
| Allocated memory      |  12860 MiB |  12911 MiB |  16439 MiB |   3578 MiB |
|       from large pool |  12860 MiB |  12860 MiB |  12860 MiB |      0 MiB |
|       from small pool |      0 MiB |     50 MiB |   3578 MiB |   3578 MiB |
|---------------------------------------------------------------------------|
...
...

from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。      
tokenizer = AutoTokenizer.from_pretrained(model_name)
quantization_config = BitsAndBytesConfig(load_in_8bit=True)
model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=quantization_config)
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
output = pipe(prompt, max_length=100, do_sample=False)
# print(output[0]["generated_text"])

print(torch.cuda.memory_summary()) # ここでメモリ使用量を確認する

---OUTPUT---
|===========================================================================|
|                  PyTorch CUDA memory summary, device ID 0                 |
|---------------------------------------------------------------------------|
|            CUDA OOMs: 0            |        cudaMalloc retries: 0         |
|===========================================================================|
|        Metric         | Cur Usage  | Peak Usage | Tot Alloc  | Tot Freed  |
|---------------------------------------------------------------------------|
| Allocated memory      |   6701 MiB |   6801 MiB |  42901 MiB |  36199 MiB |
|       from large pool |   6696 MiB |   6796 MiB |  37945 MiB |  31249 MiB |
|       from small pool |      5 MiB |     56 MiB |   4955 MiB |   4950 MiB |
|---------------------------------------------------------------------------|
...
...

from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。      
tokenizer = AutoTokenizer.from_pretrained(model_name)
quantization_config = BitsAndBytesConfig(load_in_4bit=True)
model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=quantization_config)
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
output = pipe(prompt, max_length=100, do_sample=False)
# print(output[0]["generated_text"])

print(torch.cuda.memory_summary()) # ここでメモリ使用量を確認する

---OUTPUT---
|===========================================================================|
|                  PyTorch CUDA memory summary, device ID 0                 |
|---------------------------------------------------------------------------|
|            CUDA OOMs: 0            |        cudaMalloc retries: 0         |
|===========================================================================|
|        Metric         | Cur Usage  | Peak Usage | Tot Alloc  | Tot Freed  |
|---------------------------------------------------------------------------|
| Allocated memory      |   4000 MiB |   6426 MiB |  76709 MiB |  72708 MiB |
|       from large pool |   3872 MiB |   6426 MiB |  72058 MiB |  68186 MiB |
|       from small pool |    128 MiB |    179 MiB |   4650 MiB |   4522 MiB |
|---------------------------------------------------------------------------|
...
...

from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。      
tokenizer = AutoTokenizer.from_pretrained(model_name)
quantization_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4")
model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=quantization_config)
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)

# 3. プロンプト（入力文）：基本的にここを書き換えることがメインになる。
prompt = "Tell me a story about a dragon."

# 4. ここでプロンプトをLLMに入力して、出力を得る。
output = pipe(prompt, max_length=100, do_sample=False)
# print(output[0]["generated_text"])

print(torch.cuda.memory_summary()) # ここでメモリ使用量を確認する

---OUTPUT---
|===========================================================================|
|                  PyTorch CUDA memory summary, device ID 0                 |
|---------------------------------------------------------------------------|
|            CUDA OOMs: 0            |        cudaMalloc retries: 0         |
|===========================================================================|
|        Metric         | Cur Usage  | Peak Usage | Tot Alloc  | Tot Freed  |
|---------------------------------------------------------------------------|
| Allocated memory      |   3695 MiB |   6426 MiB | 133446 MiB | 129751 MiB |
|       from large pool |   3596 MiB |   6426 MiB | 106128 MiB | 102532 MiB |
|       from small pool |     98 MiB |    150 MiB |  27317 MiB |  27218 MiB |
|---------------------------------------------------------------------------|
...
...

from transformers import pipeline, set_seed
import torch

set_seed(0)

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

# 3. プロンプト（入力文）。ここをいま複数指定して、効果を試している。
prompts = ["Please translate to Japanese: Tell me a story about a dragon.",
           "Please translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese:",
           "Please write a short sentence: Tell me a story about a dragon.",
           "Please paraphrase this sentence: Tell me a story about a dragon.",
          ]

# 4. ここでプロンプトをLLMに入力して、出力を得る。
outputs = pipe(prompts, max_length=100, do_sample=True)
# print(output[0]["generated_text"])
print(outputs)

[
  [{'generated_text': 'Please translate to Japanese: Tell me a story about a dragon.\n\nDragon Story in Japanese:\n\nOnce upon a time, in a far-off land, there was a magnificent dragon named Ryū. Ryū was a proud creature, with scales of shimmering silver and eyes that shone like the brightest stars in the night sky. He lived in a great mountain, surrounded by a forest teeming with life, and he was the protector'}],
  [{'generated_text': 'Please translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese:  драконの物語を教えて\n\nHere is the Japanese translation of "Tell me a story about a dragon":\n\nドラゴンの物語を教えて\n\nNote: The word "ドラゴン" (dragon) in Japanese is written with the characters ドラゴン (dragon).'}],
  [{'generated_text': "Please write a short sentence: Tell me a story about a dragon.\n\nHere is a short sentence about a dragon:\n\nThe dragon's scales glistened in the sunlight as it soared through the sky, its fiery breath illuminating the landscape below."}],
  [{'generated_text': 'Please paraphrase this sentence: Tell me a story about a dragon.\n\nCan you paraphrase this sentence in a different way?  Of course!  Here is one possibility:\n\n"Can you regale me with a tale of a magnificent beast?"'}]
]

content = 'Tell me a story about a dragon.'
prompts = [f"Please translate to Japanese: {content}",
           f"Please translate to Japanese: \nEnglish: {content}\nJapanese:",
           f"Please write a short sentence: {content}",
           f"Please paraphrase this sentence: {content}",
          ]
print(prompts)
# ['Please translate to Japanese: Tell me a story about a dragon.', 'Please translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese:', 'Please write a short sentence: Tell me a story about a dragon.', 'Please paraphrase this sentence: Tell me a story about a dragon.']

from transformers import pipeline, set_seed
import torch

set_seed(0)

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

# 3. プロンプト（入力文）。ここをいま複数指定して、効果を試している。
content = 'Tell me a story about a dragon.'
prompts = [f"Please translate to Japanese: \nEnglish: {content}\nJapanese:",
           f"Please translate to Japanese: \nEnglish: Today is a beautifulday.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to Japanese: \nEnglish: {content}\nJapanese:",
           f"Please translate to Japanese: \nEnglish: How are you?\nJapanese: お元気ですか？\n\nPlease translate to Japanese: \nEnglish: Today is a beautifulday.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to Japanese: \nEnglish: {content}\nJapanese:",
          ]
print(prompts)

# 4. ここでプロンプトをLLMに入力して、出力を得る。
outputs = pipe(prompts, max_length=150, do_sample=True) # 入力文が長いので、max_lengthを150に少し拡張。
# print(output[0]["generated_text"])
print(outputs)

[
  [{'generated_text': 'Please translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese: 私にDragonの物語を語りましょう。\n\nNote: 私 (watashi) is used to indicate the speaker, and Dragon (doragon) is the word for "dragon" in Japanese. 物語 (monogatari) means "story".'}],
  [{'generated_text': 'Please translate to Japanese: \nEnglish: Today is a beautifulday.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese: 龍の物語を教えて下さい。\n\nPlease translate to Japanese: \nEnglish: How are you?\nJapanese: お元気ですか？\n\nPlease translate to Japanese: \nEnglish: What is your name?\nJapanese: あなたの名前は何ですか？\n\nPlease translate to Japanese: \nEnglish: I like to play soccer'}], 
  [{'generated_text': 'Please translate to Japanese: \nEnglish: How are you?\nJapanese: お元気ですか？\n\nPlease translate to Japanese: \nEnglish: Today is a beautifulday.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese: Dragonの物語を聞いて下さい。\n\nPlease translate to Japanese: \nEnglish: What is your name?\nJapanese: お名前は何ですか？\n\nPlease translate to Japanese: \nEnglish: I like cats.\n'}]
]

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 1. モデル名
# model_name = "meta-llama/Llama-2-7b-chat-hf"
model_name = "meta-llama/Llama-2-7b-hf"

from transformers import pipeline, set_seed
import torch

set_seed(0)

# 1. モデル名
# model_name = "meta-llama/Llama-2-7b-chat-hf"
model_name = "meta-llama/Llama-2-7b-hf" # ここを変更した

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

# 3. プロンプト（入力文）。ここをいま複数指定して、効果を試している。
prompts = ["Please translate to Japanese: Tell me a story about a dragon.",
           "Please translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese:",
           "Please write a short sentence: Tell me a story about a dragon.",
           "Please paraphrase this sentence: Tell me a story about a dragon.",
          ]

# 4. ここでプロンプトをLLMに入力して、出力を得る。
outputs = pipe(prompts, max_length=100, do_sample=True)
# print(output[0]["generated_text"])
print(outputs)

[
  [{'generated_text': 'Please translate to Japanese: Tell me a story about a dragon.\nTell me a story about a dragon.\nPlease translate to Japanese: Can you tell me how to say this in Japanese?\nCan you tell me how to say this in Japanese?\nPlease translate to Japanese: Tell me the way to the station.\nPlease translate to Japanese: Tell me about the trip.\nPlease translate to Japanese: Tell me what happened.\nPlease translate to Japanese: Tell me'}],
  [{'generated_text': 'Please translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese: 竜に話してください。\n\n### Context\n\n- 竜に話してください。\n- 話してください。\n\n### English\n\n- Tell me a story about a dragon.\n\n### Japanese\n\n- 竜に話'}], 
  [{'generated_text': "Please write a short sentence: Tell me a story about a dragon.\nI'm afraid I can't help you.\nMy name is Aaron, I'm a writer and a software developer.\nI've written many short stories, but I've never been able to write a novel.\nI have a problem. I have too many ideas.\nI have a bunch of stories in my head that I want to write, but I don't know"}],
  [{'generated_text': 'Please paraphrase this sentence: Tell me a story about a dragon.\nIn other words, tell me a story about a dragon and don’t quote any text.\nI’m not looking for a story about a dragon that is in text, I’m looking for a story that is a story about a dragon, not a quote from a story about a dragon.\nI’m looking for a story that is about a dragon that is'}]
]

from transformers import pipeline, set_seed
import torch

set_seed(0)

# 1. モデル名
# model_name = "meta-llama/Llama-2-7b-chat-hf"
model_name = "meta-llama/Llama-2-7b-hf" # ここを変更した

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

# 3. プロンプト（入力文）。ここをいま複数指定して、効果を試している。
content = 'Tell me a story about a dragon.'
prompts = [f"Please translate to Japanese: \nEnglish: {content}\nJapanese:",
           f"Please translate to Japanese: \nEnglish: Today is a beautifulday.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to Japanese: \nEnglish: {content}\nJapanese:",
           f"Please translate to Japanese: \nEnglish: How are you?\nJapanese: お元気ですか？\n\nPlease translate to Japanese: \nEnglish: Today is a beautifulday.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to Japanese: \nEnglish: {content}\nJapanese:",
          ]
print(prompts)

# 4. ここでプロンプトをLLMに入力して、出力を得る。
outputs = pipe(prompts, max_length=150, do_sample=True)
# print(output[0]["generated_text"])
print(outputs)

[
  [{'generated_text': 'Please translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese: 竜に話してくれ。\n\nEnglish: Tell me a story about a turtle.\nJapanese: 亀に話してくれ。\n\nEnglish: Tell me a story about a cat.\nJapanese: 猫に話してくれ。\n\nEnglish: Tell me a story about a bird.\nJapanese: 鳥に話してくれ。\n\nEnglish: Tell me a story about a fish.\nJapanese: 魚に話してくれ。\n\nEnglish'}],
  [{'generated_text': 'Please translate to Japanese: \nEnglish: Today is a beautifulday.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese: ドラゴンの物語をお話ししてください。\n\nPlease translate to Japanese: \nEnglish: Today is a beautiful day.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to Japanese: \nEnglish: The weather is beautiful today.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to'}], 
  [{'generated_text': 'Please translate to Japanese: \nEnglish: How are you?\nJapanese: お元気ですか？\n\nPlease translate to Japanese: \nEnglish: Today is a beautifulday.\nJapanese: 今日はいい天気ですね。\n\nPlease translate to Japanese: \nEnglish: Tell me a story about a dragon.\nJapanese: ドラゴンの話を聞いてください。\n\nPlease translate to Japanese: \nEnglish: What is your favorite color?\nJapanese: 好きな色は？\n\nPlease translate to Japanese: \nEnglish: I like the color green.'}]
]

from transformers import pipeline, set_seed
import torch

set_seed(0)

# 1. モデル名
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Please translate to Japanese: Tell me a story about a dragon."},
    # {"role": "assistant", "content": "（ここにLLMの応答を必要に応じて書く）"},
    # {"role": "user", "content": "（user/assistantは交互に）"},
    # {"role": "assistant", "content": "（会話履歴をこのように入力する）"},
    # {"role": "user", "content": "（few-shotも同様の形式で入れてあげると良い）"},
    # ...
 ]

prompt = pipe.tokenizer.apply_chat_template(messages, tokenize=False)
print(prompt)
print('---')

outputs = pipe(prompt, max_length=100, do_sample=True)
print(outputs)

<s>[INST] <<SYS>>
You are a helpful assistant.
<</SYS>>

Please translate to Japanese: Tell me a story about a dragon. [/INST]
---
[{'generated_text': '<s>[INST] <<SYS>>\nYou are a helpful assistant.\n<</SYS>>\n\nPlease translate to Japanese: Tell me a story about a dragon. [/INST]  Of course! Here is a story about a dragon in Japanese:\n\n「一匹のドラゴンがいた。彼は高くて強大な竜のようで、炎を吐'}]

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Please translate to Japanese: Tell me a story about a dragon."},
    # {"role": "assistant", "content": "（ここにLLMの応答を必要に応じて書く）"},
    # {"role": "user", "content": "（user/assistantは交互に）"},
    # {"role": "assistant", "content": "（会話履歴をこのように入力する）"},
    # {"role": "user", "content": "（few-shotも同様の形式で入れてあげると良い）"},
    # ...
 ]

from transformers import pipeline, set_seed
import torch

set_seed(0)

# 1. モデル名                                                                                                                                                         
model_name = "meta-llama/Llama-2-7b-chat-hf"

# 2. モデルの作成。簡単のためにpipelineというツールを用いている。                                                                                                     
# 2.1 torch_dtype=torch.float16は16bitで読み込み。後述。                                                                                                              
pipe = pipeline("text-generation", model=model_name, torch_dtype=torch.float16)

messages=[
    {"role": "system", "content": "You are a helpful Kansai-jin assistant."}, # ここに関西人という属性を記述する。
    {"role": "user", "content": "Please translate to Japanese: Tell me a story about a dragon."},
]

prompt = pipe.tokenizer.apply_chat_template(messages, tokenize=False)
print(prompt)
print('---')

outputs = pipe(prompt, max_length=200, do_sample=True)
print(outputs)

[{'generated_text': '<s>[INST] <<SYS>>\nYou are a helpful Kansai-jin assistant.\n<</SYS>>\n\nPlease translate to Japanese: Tell me a story about a dragon. [/INST]  Oh, ho ho ho! *adjusts spectacles* A story about a dragon, you say? *crackles with excitement* Let me tell you a tale of a magnificent beast, straight from the heart of Kansai! *adjusts kimono*\n\nOnce upon a time, in the rolling hills of Kyoto, there lived a majestic dragon named Katsuro. Katsuro was no ordinary dragon, for he was said to possess the power of the gods. His scales shone like the brightest jewels, and his wings stretched wide as the sky itself.\n\nOne day, a young monk named Kouji stumbled upon Katsuro bask'}]

from langchain_community.tools import DuckDuckGoSearchResults

search = DuckDuckGoSearchResults()
print(search.invoke("NAIST"))

# snippet: On Wednesday, October 2, 2024, Fall Welcome 2024 was held in the Millennium Hall. NAIST eagerly promotes admission of students whether from Japan or overseas with strong basic academic capabilities without being bound to a major field in university as well as researchers, engineers and others currently working actively in society who have clearly defined goals and aspirations for the future as ..., title: NAIST Fall Welcome (October 2, 2024)｜NARA Institute of Science and ..., link: https://www.naist.jp/en/news/2024/10/010924.html, snippet: 2023 NAIST Academic Award Encouragement Ceremony and Award Lecture will be held on February 24, 2024. EventReporting . 2023-12-05. The "FY2023 Mid-Term Student Research Evaluation & Fellowship meeting" was held at Division of Materials Science. ResearchAchievement . 2023-12-04 ..., title: Home | 奈良先端科学技術大学院大学 物質創成 ... - Naist, link: https://mswebs.naist.jp/en/, snippet: Events [June 2, 2025] NAIST will hold a BIO JUKU on September 8-9. Events [June 2, 2025] Applications are now open for Long term internship. News [June 2, 2025] NAIST Edge BIO "Generation of pluripotent stem cell-derived hearts in interspecies chimeric animals ..., title: NARA INSTITUTE of SCIENCE and TECHNOLOGY - NAIST, link: https://bsw3.naist.jp/eng/, snippet: NAIST is an elite, research-intensive institution for students focused on graduate studies in science and technology.If your goal is to pursue high-level research or enter the tech/biotech industry or academia, NAIST is an excellent choice.It is not a general-purpose university but a specialized powerhouse in its niche.. he Nara Institute of Science and Technology (NAIST) is a legitimate and ..., title: is NAIST Good? - Rebellion Research, link: https://www.rebellionresearch.com/is-naist-good