Merge pull request #26 from CTUbase/feat/ai

Add ai api

Author: NguyenDoanHoangPhuc

ai_api/__pycache__/analyze_data.cpython-312.pyc

@@ -0,0 +1,135 @@
+import requests
+from pyvi import ViTokenizer, ViPosTagger
+import string
+from utils.utils import *
+from utils.crawl import *
+
+def create_tokens(doc) -> list:
+    doc = ViTokenizer.tokenize(doc) 
+    doc = doc.lower() 
+    tokens = doc.split()
+    table = str.maketrans('', '', string.punctuation.replace("_", "")) 
+    tokens = [w.translate(table) for w in tokens]
+    tokens = [word for word in tokens if word]
+    stopwords = get_tokens_from_file('./sample/stopwords.txt')
+    tokens = [word for word in tokens if word not in stopwords]
+    return tokens
+
+
+
+
+def count_tokens(check_tokens, tokens):
+    count = 0
+    for token in tokens:
+        if token in check_tokens:
+            count += 1
+    return count
+
+def count_partial_tokens(check_tokens, tokens):
+    count = 0
+    for token in tokens:
+        for check_token in check_tokens:
+            if check_token in token:
+                count += 1
+                break  # Stop checking other check_tokens for this token
+    return count
+
+def first_appearance_index(test_token_list, context):
+    for idx, test_token in enumerate(test_token_list):
+        count = count_tokens(test_token, context)
+        if count > 0:
+            return idx+1
+    return 0
+
+def appearance_array(test_token_list, context):
+    appearance = []
+    for test_token in test_token_list:
+        count = count_tokens(test_token, context)
+        appearance.append(count)
+    return appearance
+
+
+def remove_tokens(tokens, check_tokens):
+    for token in tokens:
+        if token in check_tokens:
+            tokens.remove(token)
+    return tokens
+
+
+def create_input(article):
+
+    # Lấy token từ url tương ứng trên repo github
+    people_death_tokens = get_tokens_from_file('./sample/people_death_keywords.txt')
+    people_injuries_tokens = get_tokens_from_file('./sample/people_injuries_keywords.txt')
+    large_property_tokens = get_tokens_from_file('./sample/property_large_keywords.txt')
+    average_property_tokens = get_tokens_from_file('./sample/property_medium_keywords.txt')
+    small_property_tokens = get_tokens_from_file('./sample/property_small_keywords.txt')
+    serious_keyword_tokens = get_tokens_from_file('./sample/serious_keywords.txt')
+
+    # Khởi tạo dữ liệu ban đầu
+    data = {
+        'deaths': 0,
+        'injuries': 0,
+        'property': [],
+        'keyword': 0
+    }
+    
+    tokens = create_tokens(article)
+
+    # Xử lý số
+    number_conversion = {
+        "trăm": 100,
+        "ngàn": 1000,
+        "nghìn": 1000,
+        "vạn": 10000,
+        "triệu": 1000000,
+        "tỷ": 1000000000,
+        "trăm_ngàn": 100000,
+        "chục_ngàn": 10000,
+        "chục": 10,
+        "mươi": 10,
+        "nhiều": 2
+    }
+
+    for idx, token in enumerate(tokens):
+        if token in number_conversion:
+            tokens[idx] = str(number_conversion[token])
+
+
+    people_token_list = [people_death_tokens, people_injuries_tokens]
+    property_token_list = [large_property_tokens, average_property_tokens, small_property_tokens]
+
+    for token in tokens:
+
+        if token.isdigit():
+            left_context = tokens[max(0, tokens.index(token) - 2):tokens.index(token)]
+            check_value = first_appearance_index(people_token_list, left_context)
+            if (check_value != 0):
+                if check_value == 1:
+                    data['deaths'] += int(token)
+                else:
+                    data['injuries'] += int(token)
+                remove_tokens(tokens, left_context)
+                continue
+
+
+            right_context = tokens[tokens.index(token) + 1:tokens.index(token) + 2]
+            check_value = first_appearance_index(people_token_list, right_context)
+            if (check_value != 0):
+                if check_value == 1:
+                    data['deaths'] += int(token)
+                else:
+                    data['injuries'] += int(token)
+                remove_tokens(tokens, right_context)
+                continue
+
+
+
+    check_value = appearance_array(property_token_list, tokens)
+    data['property'] = check_value
+
+    filtered_tokens = [word for word in tokens if len(word) > 2]
+    length = len(filtered_tokens)
+    data['keyword'] = count_tokens(serious_keyword_tokens, filtered_tokens)/length
+
+    return data

ai_api/__pycache__/perceptron.cpython-312.pyc

@@ -0,0 +1,85 @@
+# app.py
+from fastapi import FastAPI
+from pydantic import BaseModel
+from utils.crawl import *
+from utils.analyze_data import *
+import joblib
+from joblib import Parallel, delayed
+from analyze_data import *
+import numpy as np
+import pandas as pd
+from perceptron import Perceptron
+
+app = FastAPI()
+
+# Khai báo mẫu request data
+class InputData(BaseModel):
+    keyword: str
+    num_pages: int
+
+@app.post("/getinfo")
+def run_prediction(data: InputData):
+    data = find_vnexpress(data.keyword, data.num_pages)
+
+    model = joblib.load('models/trained_perceptron_model.joblib')
+
+    location_urls = {}
+    def process_url(url):
+
+        analyzed_data = analyze(url)
+        if analyzed_data is not None:
+            prediction = model.predict([analyzed_data['ratio']])
+            if prediction[0] == 1:
+                locations = find_location(analyzed_data['article'])
+                return url, locations
+        return url, []
+
+    data = Parallel(n_jobs=-1)(delayed(process_url)(url) for url in data)
+
+    
+    location_to_url = {}
+    for url, locations in data:
+        for loc in locations:
+            # Chỉ thêm nếu location chưa từng xuất hiện
+            if loc not in location_to_url:
+                location_to_url[loc] = url
+
+    # Chuyển dict sang list (location, url)
+    result = [(loc, url) for loc, url in location_to_url.items()]
+    return result
+
+@app.get("/ping")
+def ping():
+    return "pong"
+
+
+class InputData1(BaseModel):
+    article: str
+
+@app.post("/predict")
+def predict(data: InputData1):
+    article = data.article
+    data = create_input(article)
+
+    model = joblib.load('models/best_decision_tree_model.joblib')
+
+    data_df = pd.DataFrame([{
+        'death': data['deaths'],
+        'injuries': data['injuries'],
+        'property_large': data['property'][0],
+        'property_medium': data['property'][1],
+        'property_small': data['property'][2],
+        'keyword': data['keyword']
+    }])
+
+    # Nếu bạn muốn trả về DataFrame cho mục đích debug
+    result = data_df.to_dict(orient="records")
+
+    # Ép kiểu dữ liệu numpy.int64 về int (nếu có)
+    for row in result:
+        for k, v in row.items():
+            if isinstance(v, np.integer):
+                row[k] = int(v)
+
+    prediction = model.predict(data_df)
+    return int(prediction[0])

ai_api/analyze_data.py

@@ -0,0 +1,34 @@
+import csv
+from utils.crawl import *
+from utils.utils import * 
+from utils.analyze_data import *
+
+with open('input.txt', 'r') as file:
+    urls = [line.strip() for line in file.readlines()]
+
+for url in urls: 
+    # Crawl dữ liệu từ url
+    
+    data = analyze(url)
+
+    article = data['article']
+    ratios = data['ratio']
+
+    # Write the ratios to a CSV file
+    with open('datasets/datasets.csv', 'a', newline='', encoding='utf-8') as csvfile:
+        fieldnames = ['region', 'bao_lu', 'dich_benh', 'label']
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+
+        # Write the header only if the file is empty
+        if csvfile.tell() == 0:
+            writer.writeheader()
+
+        writer.writerow({
+            'region': ratios[0],
+            'bao_lu': ratios[1],
+            'dich_benh': ratios[2],
+            'label': 0
+        })
+    
+print("Succcessfully write datasets!")
+

ai_api/app.py

@@ -0,0 +1,108 @@
+region,bao_lu,dich_benh,label
+0.0056179775280898,0.1910112359550561,0.0926966292134831,0
+0.0539215686274509,0.1470588235294117,0.1372549019607843,0
+0.1925133689839572,0.0641711229946524,0.213903743315508,1
+0.0824742268041237,0.134020618556701,0.134020618556701,1
+0.0048076923076923,0.0528846153846153,0.0240384615384615,0
+0.0210526315789473,0.0912280701754386,0.2245614035087719,0
+0.1778523489932885,0.2080536912751678,0.1006711409395973,1
+0.0273972602739726,0.1598173515981735,0.0593607305936073,0
+0.1794019933554817,0.2491694352159468,0.0631229235880398,1
+0.0045045045045045,0.0945945945945946,0.063063063063063,0
+0.0608695652173913,0.2260869565217391,0.0695652173913043,1
+0.0054644808743169,0.0983606557377049,0.081967213114754,0
+0.0572687224669603,0.092511013215859,0.2819383259911894,1
+0.1212121212121212,0.0484848484848484,0.2121212121212121,1
+0.1506849315068493,0.2547945205479452,0.1013698630136986,1
+0.0186046511627906,0.0542635658914728,0.2403100775193798,0
+0.0303951367781155,0.1580547112462006,0.2796352583586626,1
+0.0258620689655172,0.2327586206896551,0.0560344827586206,0
+0.0216606498194945,0.1028880866425992,0.2220216606498194,0
+0.0878661087866108,0.209205020920502,0.0753138075313807,1
+0.0834724540901502,0.0684474123539232,0.0400667779632721,0
+0.1010719754977029,0.1822358346094946,0.0704441041347626,1
+0.1780821917808219,0.0639269406392694,0.2146118721461187,1
+0.0602409638554216,0.2048192771084337,0.0602409638554216,1
+0.0631229235880398,0.1428571428571428,0.0398671096345514,1
+0.0040485829959514,0.1781376518218623,0.0607287449392712,0
+0.0022172949002217,0.1529933481152993,0.0709534368070953,0
+0.0573770491803278,0.1967213114754098,0.0163934426229508,1
+0.0108695652173913,0.1304347826086956,0.0326086956521739,0
+0.0262295081967213,0.1081967213114754,0.2950819672131147,1
+0.0102389078498293,0.1331058020477815,0.0238907849829351,0
+0.0246913580246913,0.2037037037037037,0.0802469135802469,0
+0.08,0.136,0.08,1
+0.0465116279069767,0.0598006644518272,0.2624584717607973,1
+0.0713324360699865,0.2395693135935397,0.0592193808882907,1
+0.2705570291777188,0.0928381962864721,0.1777188328912466,1
+0.0226876090750436,0.0907504363001745,0.1884816753926701,0
+0.0446428571428571,0.1860119047619047,0.0982142857142857,0
+0.0115942028985507,0.2057971014492753,0.1130434782608695,0
+0.2697674418604651,0.2162790697674418,0.127906976744186,1
+0.1524008350730689,0.1962421711899791,0.0855949895615866,1
+0.076595744680851,0.251063829787234,0.1617021276595744,1
+0.1183294663573085,0.2923433874709977,0.1345707656612529,1
+0.050314465408805,0.1488469601677149,0.2075471698113207,0
+0.0150093808630394,0.0769230769230769,0.0675422138836773,0
+0.211864406779661,0.173728813559322,0.1101694915254237,1
+0.0465686274509803,0.196078431372549,0.0318627450980392,1
+0.1142191142191142,0.2773892773892774,0.0932400932400932,1
+0.0249110320284697,0.1423487544483986,0.2348754448398576,1
+0.0171428571428571,0.0714285714285714,0.2342857142857143,0
+0.0071428571428571,0.2642857142857143,0.0571428571428571,0
+0.0155902004454342,0.1648106904231625,0.0534521158129175,0
+0.1243144424131627,0.1992687385740402,0.1371115173674588,1
+0.0050251256281407,0.1206030150753768,0.1055276381909547,1
+0.1380368098159509,0.0613496932515337,0.2239263803680981,1
+0.0818858560794044,0.1910669975186104,0.0446650124069478,1
+0.0225225225225225,0.1171171171171171,0.0315315315315315,0
+0.0710382513661202,0.1475409836065573,0.0655737704918032,1
+0.0163265306122449,0.1306122448979592,0.0775510204081632,0
+0.0833333333333333,0.1842105263157894,0.0745614035087719,1
+0.0,0.0215827338129496,0.0287769784172661,0
+0.0577427821522309,0.1181102362204724,0.0866141732283464,1
+0.1147540983606557,0.2513661202185792,0.1912568306010929,1
+0.0681818181818181,0.1409090909090909,0.0727272727272727,0
+0.0454545454545454,0.0848484848484848,0.109090909090909,0
+0.1872509960159362,0.0796812749003984,0.095617529880478,1
+0.3027522935779816,0.0672782874617737,0.1376146788990825,1
+0.0656716417910447,0.2835820895522388,0.1940298507462686,1
+0.0489296636085626,0.0764525993883792,0.0397553516819571,0
+0.15,0.175,0.0416666666666666,1
+0.1142191142191142,0.2773892773892774,0.0932400932400932,1
+0.0620272314674735,0.2602118003025718,0.0423600605143721,1
+0.0234234234234234,0.2108108108108108,0.0342342342342342,0
+0.0,0.0526315789473684,0.0526315789473684,0
+0.0546875,0.1015625,0.0390625,0
+0.1338289962825278,0.1078066914498141,0.1672862453531598,1
+0.0588235294117647,0.1176470588235294,0.0647058823529411,0
+0.0437956204379562,0.2992700729927007,0.0510948905109489,1
+0.0496453900709219,0.226950354609929,0.0319148936170212,0
+0.2696335078534031,0.0785340314136125,0.1544502617801047,1
+0.0397553516819571,0.1345565749235474,0.2996941896024465,1
+0.0,0.0918918918918919,0.1243243243243243,0
+0.0744680851063829,0.2279635258358662,0.0623100303951367,1
+0.0477326968973747,0.2529832935560859,0.1360381861575179,0
+0.0196078431372549,0.2794117647058823,0.088235294117647,0
+0.0583333333333333,0.225,0.0916666666666666,1
+0.0237580993520518,0.0302375809935205,0.0367170626349892,0
+0.026615969581749,0.0342205323193916,0.2585551330798479,0
+0.0730337078651685,0.2247191011235955,0.1404494382022472,1
+0.0682730923694779,0.0642570281124498,0.0040160642570281,0
+0.0161527165932452,0.1468428781204111,0.1071953010279001,1
+0.044776119402985,0.0762852404643449,0.2255389718076285,0
+0.0546875,0.1015625,0.0390625,0
+0.0057471264367816,0.0459770114942528,0.1408045977011494,0
+0.0585480093676815,0.0983606557377049,0.0140515222482435,0
+0.0699588477366255,0.1440329218106996,0.1810699588477366,0
+0.1071428571428571,0.0357142857142857,0.0892857142857142,0
+0.0394736842105263,0.2467105263157894,0.0361842105263157,1
+0.056390977443609,0.1578947368421052,0.1165413533834586,0
+0.0109890109890109,0.1062271062271062,0.1684981684981685,0
+0.0289017341040462,0.0809248554913294,0.3121387283236994,1
+0.029090909090909,0.2945454545454545,0.069090909090909,1
+0.0772946859903381,0.1473429951690821,0.0893719806763285,1
+0.0111940298507462,0.1791044776119403,0.0634328358208955,0
+0.0729166666666666,0.2125,0.0875,1
+0.0,0.0817610062893081,0.1446540880503144,0
+0.0319148936170212,0.2712765957446808,0.0292553191489361,1

ai_api/create_datasets.py

@@ -0,0 +1,17 @@
+FROM python:3.9-slim
+
+# Thiết lập thư mục làm việc
+WORKDIR /app
+
+# Sao chép file requirements
+COPY requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Sao chép toàn bộ code
+COPY . /app
+
+# Mở port
+EXPOSE 9000
+
+# Lệnh chạy API bằng uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "9000"]

ai_api/datasets/datasets.csv

@@ -0,0 +1 @@
+https://vnexpress.net/con-bao-toc-do-560-km-moi-gio-tren-sao-moc-4163572.html