【使用 BERT 的问答系统】第 6 章 ：BERT 模型应用：其他任务

from deeppavlov import build_model, configs
def build_sentiment_model ():
    model = build_model(configs.classifiers.insults_kaggle_bert, download=True)
    return model
test_input = ['This movie is good', 'You are so dumb!']
if __name__ == "__main__" :
        sentiment_model = build_sentiment_model()
        results = sentiment_model(test_ input)
        print(results)

from flask import Flask, request
import json
from SentimentAnalysis.SentimentAnalysis import build_sentiment_model
app=Flask(__name__)
@app.route ("/sentimentAnalysis", methods=['POST'])
def sentimentAnalysis():
    try:
        json_data = request.get_json(force=True)
        questions = json_data['questions']
        sentiment_model = build_sentiment_model()
        questions_list =[]
        for ques in questions:
            questions_list.append(ques)
        model_output = sentiment_model(questions_list)
        index = 0
        result = []
        for ans in model_output:
            sentiment_qa =dict()
            sentiment_qa['qustion'] = questions_list[index]
            sentiment_qa['answer'] = ans
            result.append(sentiment_qa)
        result={'results':result}
        result = json.dumps(result)
        return result
except Exception as e:
        return {"Error": str(e)}
if __name__ == "__main__" :
    app.run(port="5000")

from deeppavlov import build_model, configs
import pandas as pd
def build_ner_model ():
    model = build_model(configs. ner.ner_ontonotes_bert_mult, download=True)
    return model
if __name__=="__main__":
  test_input = ["Amazon rainforests are located in South America."]
   ner_model = build_ner_model()
   results = ner_model(test_ input)
   results = pd.DataFrame(zip(results[0][0],results[1][0]), columns=['Word','Entity'])
   print(results)

from flask import Flask, request
import json
from NamedEntityRecognition.NamedEntityRecognition import build_ner_model
app=Flask(__name__)
@app.route ("/namedEntity", methods=['POST'])
def namedEntity():
    try:
         json_data = request.get_json(force=True)
        query = json_data['query']
        ner_model = build_ner_model()
        model_output = ner_model([query])
        words= model_output[0][0]
        tags = model_output[1][0]
        result_json = dict()
        result_json['query'] = query
        entities = []
        index = 0
        for word in words:
            word_tag_dict=dict()
            word_tag_dict['word'] = word
            word_tag_dict['tag'] = tags[index]
            index = index+1
            entities.append(word_tag_dict)
        result_json['entities'] = entities
        result = json.dumps(result_json)
        return result
except Exception as e:
        return {"Error": str(e)}
if __name__ == "__main__" :
    app.run(port="5000")

from sklearn.datasets import fetch_20newsgroups
import ktrain
from ktrain import text
def preprocess_dataset():
    classes = ['alt.atheism', 'soc.religion.christian','comp.graphics', 'sci.med']
    train_data = fetch_20newsgroups(subset='train', categories=classess, shuffle=True, random_state=42)
    test_data = fetch_20newsgroups(subset='test', categories=classes, shuffle=True, random_state=42)
    return train_data.data,train_data.target, test_data.data,test_data.target,classes
def create_text_classification_model():
    MODEL_NAME = 'distilbert-base-uncased'
    train_features, train_labels, test_features, test_labels, train_classes =preprocess_dataset()
    trans = text.Transformer(MODEL_NAME, maxlen=500, classes=train_classes)
    train_preprocess = trans.preprocess_train(train_features, train_labels)
    val_preprocess = trans.preprocess_test(test_features, test_labels)
    model_data = trans.get_classifier()
    classification_model = ktrain.get_learner(model_data, train_data=train_preprocess, val_data=val_preprocess, batch_size=6)
    classification_model.fit_onecycle(5e-5, 4)
    return classification_model, trans
def predict_category(classification_model, trans, input_text):
    predictor = ktrain.get_predictor(classification_model.model, preproc=trans)
    results = predictor.predict(input_text)
    return results
if __name__ == "__main__" :
        classification_model, trans = create_text_classification_model()
        input_text = 'Babies with down syndrome have an extra chromosome.'
        print(predict_category(classification_model, trans, input_text))