# 准备语料
corpus = [“I love natural language processing”, “Natural language processing is interesting”, “I am learning natural language processing”]

# 分词
tokenized_corpus = [word_tokenize(sentence.lower()) for sentence in corpus]

# 构建n-gram模型
n = 2
train_data, padded_sents = padded_everygram_pipeline(n, tokenized_corpus)
model = NgramLanguageModeler(train_data)

# 预测下一个词
context = [“natural”, “language”]
print(“Next word predictions:”)
for word in model.vocab:
print(f”{word}: {model.score(word, context)}”)

# 英文分词
text = “I love natural language processing. It is very interesting.”
sentences = sent_tokenize(text)
print(“句子分词:”, sentences)

words = word_tokenize(text)
print(“单词分词:”, words)

# 中文分词
import jieba
chinese_text = “我爱自然语言处理。它非常有趣。”
chinese_words = jieba.cut(chinese_text)
print(“中文分词:”, list(chinese_words))

text = “I love natural language processing”
words = word_tokenize(text)
tagged_words = nltk.pos_tag(words)
print(“词性标注:”, tagged_words)

text = “Barack Obama was born in Hawaii. He was the 44th President of the United States.”
words = word_tokenize(text)
tagged_words = nltk.pos_tag(words)
named_entities = ne_chunk(tagged_words)
print(“命名实体识别:”)
print(named_entities)

# 准备语料
corpus = [“I love natural language processing”, “Natural language processing is interesting”, “I am learning natural language processing”]
tokenized_corpus = [word_tokenize(sentence.lower()) for sentence in corpus]

# 训练Word2Vec模型
model = Word2Vec(sentences=tokenized_corpus, vector_size=100, window=5, min_count=1, workers=4)

# 获取词向量
print(“Word vector for ‘natural’:”)
print(model.wv[“natural”])

# 查找相似词
print(“Words similar to ‘natural’:”)
print(model.wv.most_similar(“natural”))

# 加载预训练模型和分词器
tokenizer = BertTokenizer.from_pretrained(‘bert-base-uncased’)
model = BertForSequenceClassification.from_pretrained(‘bert-base-uncased’, num_labels=2)

# 准备输入
text = “I love natural language processing”
inputs = tokenizer(text, return_tensors=”pt”)

# 进行预测
with torch.no_grad():
outputs = model(**inputs)
logits = outputs.logits
predicted_class = torch.argmax(logits, dim=1).item()

print(“Predicted class:”, predicted_class)

# 加载预训练模型和分词器
tokenizer = GPT2Tokenizer.from_pretrained(‘gpt2’)
model = GPT2LMHeadModel.from_pretrained(‘gpt2’)

# 准备输入
prompt = “I love natural language processing because”
inputs = tokenizer(prompt, return_tensors=”pt”)

# 生成文本
outputs = model.generate(
inputs.input_ids,
max_length=50,
num_return_sequences=1,
no_repeat_ngram_size=2,
do_sample=True,
temperature=0.7
)

# 解码输出
generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(“Generated text:”, generated_text)

# 加载翻译模型
translator = pipeline(“translation_en_to_zh”)

# 翻译文本
text = “I love natural language processing”
translated_text = translator(text)
print(“翻译结果:”, translated_text[0][‘translation_text’])

# 加载情感分析模型
sentiment_analyzer = pipeline(“sentiment-analysis”)

# 分析情感
texts = [“I love natural language processing”, “I hate this movie”, “This book is okay”]
results = sentiment_analyzer(texts)
for text, result in zip(texts, results):
print(f”Text: {text}”)
print(f”Sentiment: {result[‘label’]}, Score: {result[‘score’]}”)
print()

# 加载文本摘要模型
summarizer = pipeline(“summarization”)

# 准备长文本
text = “Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence concerned with the interactions between computers and human language, in particular how to program computers to process and analyze large amounts of natural language data. Challenges in natural language processing frequently involve speech recognition, natural language understanding, natural language generation, and machine translation.”

# 生成摘要
summary = summarizer(text, max_length=50, min_length=20, do_sample=False)
print(“摘要:”, summary[0][‘summary_text’])

# 加载问答模型
question_answerer = pipeline(“question-answering”)

# 准备上下文和问题
context = “Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence concerned with the interactions between computers and human language.”
question = “What is NLP?”

# 回答问题
result = question_answerer(question=question, context=context)
print(“Answer:”, result[‘answer’])
print(“Score:”, result[‘score’])

# 下载必要的资源
nltk.download(‘punkt’)
nltk.download(‘stopwords’)
nltk.download(‘wordnet’)

# 准备文本
text = “I love natural language processing. It is very interesting and challenging.”

# 句子分词
sentences = sent_tokenize(text)
print(“句子分词:”, sentences)

# 单词分词
words = word_tokenize(text)
print(“单词分词:”, words)

# 去除停用词
stop_words = set(stopwords.words(‘english’))
filtered_words = [word for word in words if word.lower() not in stop_words]
print(“去除停用词:”, filtered_words)

# 词干提取
stemmer = PorterStemmer()
stemmed_words = [stemmer.stem(word) for word in filtered_words]
print(“词干提取:”, stemmed_words)

# 词形还原
lemmatizer = WordNetLemmatizer()
lemmatized_words = [lemmatizer.lemmatize(word) for word in filtered_words]
print(“词形还原:”, lemmatized_words)

# 加载模型
nlp = spacy.load(‘en_core_web_sm’)

# 处理文本
text = “I love natural language processing. It is very interesting and challenging.”
doc = nlp(text)

# 句子分词
print(“句子分词:”)
for sent in doc.sents:
print(sent.text)

# 单词分词和词性标注
print(“\n单词分词和词性标注:”)
for token in doc:
print(f”{token.text} – {token.pos_} – {token.lemma_}”)

# 命名实体识别
print(“\n命名实体识别:”)
for ent in doc.ents:
print(f”{ent.text} – {ent.label_}”)

# 依存关系分析
print(“\n依存关系分析:”)
for token in doc:
print(f”{token.text} → {token.dep_} → {token.head.text}”)