5分钟 NLP ：Hugging Face 主要类和函数介绍 🤗

来源：Deephub Imba本文约2200字，建议阅读9分钟本文包含其主要类和函数的概述以及一些代码示例。可以作为该库的一个入门教程 。

pip install transformers datasets

from transformers import pipeline

classifier = pipeline("sentiment-analysis")

results = classifier("I'm so happy today!")
print(f"{results[0]['label']} with score {results[0]['score']}")
# POSITIVE with score 0.9998742341995239

results = classifier(["I'm so happy today!", "I hope you don't hate him..."])
for result in results:
  print(f"{result['label']} with score {result['score']}")
# POSITIVE with score 0.9998742341995239
# NEGATIVE with score 0.6760789155960083

classifier = pipeline("sentiment-analysis",
                    model="cardiffnlp/twitter-roberta-base-sentiment",
                    tokenizer="cardiffnlp/twitter-roberta-base-sentiment")
# three possible outputs:
# LABEL_0 -> negative
# LABEL_1 -> neutral
# LABEL_2 -> positive

results = classifier(["We are very happy to show you the 🤗 Transformers library.", "We hope you don't hate it."])
for result in results:
  print(f"{result['label']} with score {result['score']}")
# LABEL_2 with score 0.9814898371696472
# LABEL_1 with score 0.5063014030456543

import datasets

dataset = datasets.load_dataset("glue", "sst2")

print(dataset)
DatasetDict({
  train: Dataset({
      features: ['sentence', 'label', 'idx'],
      num_rows: 67349
  })
  validation: Dataset({
      features: ['sentence', 'label', 'idx'],
      num_rows: 872
  })
  test: Dataset({
      features: ['sentence', 'label', 'idx'],
      num_rows: 1821
  })
})

dataset = datasets.load_dataset("glue", "sst2", split='train')

print(dataset)
Dataset({
  features: ['sentence', 'label', 'idx'],
  num_rows: 67349
})

import pandas as pd

df = pd.DataFrame(dataset)
df.head()

classifier = pipeline("sentiment-analysis")
%time results = classifier(dataset.data["sentence"].to_pylist()[:500])
# CPU times: user 21.9 s, sys: 56.9 ms, total: 22 s
# Wall time: 21.8 s

classifier = pipeline("sentiment-analysis", device=0)
%time results = classifier(dataset.data["sentence"].to_pylist()[:500])
# CPU times: user 4.07 s, sys: 49.6 ms, total: 4.12 s
# Wall time: 4.11 s

metric = datasets.load_metric("glue", "sst2")

n_samples = 500

X = dataset.data["sentence"].to_pylist()[:n_samples]
y = dataset.data["label"].to_pylist()[:n_samples]

results = classifier(X)
predictions = [0 if res["label"] == "NEGATIVE" else 1 for res in results]

print(metric.compute(predictions=predictions, references=y))
# {'accuracy': 0.988}

from transformers import AutoTokenizer, AutoModelForSequenceClassification

model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name)

encoding = tokenizer(["Hello!", "How are you?"], padding=True,
                    truncation=True, max_length=512, return_tensors="pt")

print(encoding)
{'input_ids': tensor([[ 101, 29155,   106,   102,     0,     0],
      [ 101, 12548, 10320, 10855,   136,   102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0],
      [0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 0, 0],
      [1, 1, 1, 1, 1, 1]])}

outputs = model(**encoding)

print(outputs)
SequenceClassifierOutput(loss=None, logits=tensor([[-0.2410, -0.9115, -0.3269, -0.0462, 1.2899],
      [-0.3575, -0.6521, -0.4409, 0.0471, 0.9552]],
      grad_fn=<AddmmBackward0>), hidden_states=None, attentions=None)

from torch import nn

pt_predictions = nn.functional.softmax(outputs.logits, dim=-1)

print(pt_predictions)
tensor([[0.1210, 0.0619, 0.1110, 0.1470, 0.5592],
      [0.1269, 0.0945, 0.1168, 0.1902, 0.4716]], grad_fn=<SoftmaxBackward0>)

pt_save_directory = "./model"
tokenizer.save_pretrained(pt_save_directory)
model.save_pretrained(pt_save_directory)

model = AutoModelForSequenceClassification.from_pretrained("./model")