diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..80d54f7 Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index eb8f958..f8d2159 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # ignore pycache /__pycache__/* /app/__pycache__/* +/migrations/__pycache__/* +/migrations/versions/__pycache__/* # ignore vscode /.vscode \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index cf4cd26..78fde2c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,3 @@ { - "python.pythonPath": "/anaconda3/bin/python" + "python.pythonPath": "/Users/maxalminasatriakahfi/anaconda3/bin/python" } \ No newline at end of file diff --git a/__pycache__/config.cpython-37.pyc b/__pycache__/config.cpython-37.pyc index f1e95e4..d893a1f 100644 Binary files a/__pycache__/config.cpython-37.pyc and b/__pycache__/config.cpython-37.pyc differ diff --git a/app/__pycache__/analytics.cpython-37.pyc b/app/__pycache__/analytics.cpython-37.pyc index f23f56b..1f9fb4f 100644 Binary files a/app/__pycache__/analytics.cpython-37.pyc and b/app/__pycache__/analytics.cpython-37.pyc differ diff --git a/app/__pycache__/forms.cpython-37.pyc b/app/__pycache__/forms.cpython-37.pyc index 80cb4c6..eb358da 100644 Binary files a/app/__pycache__/forms.cpython-37.pyc and b/app/__pycache__/forms.cpython-37.pyc differ diff --git a/app/__pycache__/models.cpython-37.pyc b/app/__pycache__/models.cpython-37.pyc index 1531c26..4c4d0d9 100644 Binary files a/app/__pycache__/models.cpython-37.pyc and b/app/__pycache__/models.cpython-37.pyc differ diff --git a/app/__pycache__/routes.cpython-37.pyc b/app/__pycache__/routes.cpython-37.pyc index 63dc72e..e5add6e 100644 Binary files a/app/__pycache__/routes.cpython-37.pyc and b/app/__pycache__/routes.cpython-37.pyc differ diff --git a/app/analytics.py b/app/analytics.py index 10ed43f..d1b52e1 100644 --- a/app/analytics.py +++ b/app/analytics.py @@ -9,7 +9,7 @@ import altair as alt import pandas as pd import numpy as np -import string, urllib, re, pickle +import string, urllib, re from sklearn.feature_extraction.text import TfidfVectorizer from pathlib import Path @@ -46,55 +46,30 @@ def getuserdb(): # =================================================== def get_response(): - responses = pd.read_sql_query("SELECT response.workshop_id, response.satisfaction_score, response.comments, e.id as employee_id, w.workshop_name, w.workshop_start as timestamp\ - FROM response\ - LEFT JOIN workshop w ON w.id = response.workshop_id\ - LEFT JOIN employee e ON e.id = w.workshop_instructor", conn, parse_dates='timestamp') - - responses.loc[:,'comments'] = responses['comments'].astype(str) - responses.loc[:,'satisfaction_score'] = responses['satisfaction_score'].astype(float) - + responses = pd.read_sql_query("SELECT response.workshop_id, response.satisfaction_score, response.comments, e.id as employee_id, w.workshop_name, w.workshop_start as timestamp, response.sentiment\ + FROM response\ + LEFT JOIN workshop w ON w.id = response.workshop_id\ + LEFT JOIN employee e ON e.id = w.workshop_instructor", conn, parse_dates='timestamp') sixmonths = datetime.datetime.now() - datetime.timedelta(weeks=26) responses = responses[responses.timestamp >= sixmonths] + + responses = responses[responses['comments']!='-'] + responses = responses.sort_values(by='timestamp', ascending=False).reset_index(drop=True) + return responses def get_sentiment_data(responses): - return responses[['workshop_id','workshop_name','employee_id','timestamp','comments']].copy() + return responses[['workshop_id','workshop_name','employee_id','timestamp','comments','sentiment']].copy() def get_reviews_data(responses): - return responses[['workshop_id', 'satisfaction_score', 'employee_id', 'timestamp']].copy() - -def process_sentiment(sentiment): - my_model = load(str(Path().absolute())+'/model/sentiment/model.joblib') - word_vector = load(str(Path().absolute())+'/model/sentiment/vector.joblib') - - sentiment['comments'].replace(['','None'], np.nan, inplace=True) - sentiment.dropna(inplace=True) - sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: x.lower()) - sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: x.translate(str.maketrans("","", string.punctuation))) - sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: x.translate(str.maketrans("","", string.digits))) - sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: re.sub(' +', ' ',x).strip()) - # word_vector.fit(pd.read_csv(str(Path().absolute())+'/model/sentiment/train.csv')) - - sentiment.loc[:,'score'] = '' - sentiment.loc[:,'score'] = my_model.predict(word_vector.transform(sentiment['comments'])) - - return sentiment - -def prereviews(reviews): - reviews.fillna(3, inplace=True) - - return reviews + return responses[['workshop_id','satisfaction_score','employee_id','timestamp']].copy() response = get_response() sentiment = get_sentiment_data(response) -sentiment = process_sentiment(sentiment) - reviews = get_reviews_data(response) -reviews = prereviews(reviews) domain = ['negative', 'positive'] colors = ['#7dbbd2cc', '#bbc6cbe6'] @@ -103,45 +78,44 @@ def get_params(): emp = getuserdb() dat = emp.loc[emp.this_user == True,:].copy() emp_id = dat.iloc[0]['workshop_instructor'] - sixmonths = datetime.datetime.now() - datetime.timedelta(weeks=26) - return emp_id, sixmonths + return emp_id def get_overall_sentiment(): - emp_id, sixmonths = get_params() + emp_id = get_params() person = sentiment[sentiment.employee_id==emp_id].copy() - - return pd.crosstab([person['timestamp'],person['workshop_name'],person['workshop_id']],person['score']).apply(lambda x: round(x/x.sum()*100,2), axis=1).reset_index().melt(id_vars=['timestamp','workshop_name','workshop_id']) + monyear_percent = pd.crosstab([person['timestamp'],person['workshop_name'],person['workshop_id']],person['sentiment']).apply(lambda x: round(x/x.sum()*100,2), axis=1).reset_index().melt(id_vars=['timestamp','workshop_name','workshop_id']) + + return monyear_percent.sort_values(by='timestamp', ascending=False).reset_index(drop=True) def get_overall_reviews(monyear_percent): - emp_id, sixmonths = get_params() + emp_id = get_params() filter_idx = monyear_percent.groupby(['workshop_id'])['value'].transform(max) == monyear_percent['value'] al_reviews = monyear_percent[filter_idx].copy() - al_reviews.sort_values('score', ascending=False, inplace=True) + al_reviews.sort_values('sentiment', ascending=False, inplace=True) al_reviews.drop_duplicates(subset='workshop_id', keep="first", inplace=True) - person_reviews = reviews[(reviews.employee_id==emp_id) & (reviews.timestamp >= sixmonths)] + person_reviews = reviews[(reviews.employee_id==emp_id)] - sm_reviews = person_reviews[['satisfaction_score', 'workshop_id']].groupby(['workshop_id']).mean().round(1) + sm_reviews = person_reviews[['satisfaction_score', 'workshop_id', 'timestamp']].groupby(['workshop_id','timestamp']).mean().round(1) sm_reviews.reset_index(inplace=True) sm_reviews.loc[:,'workshop_name'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['workshop_name']) - sm_reviews.loc[:,'sentiment'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['score']) + sm_reviews.loc[:,'sentiment'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['sentiment']) sm_reviews.loc[:,'value'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['value']) - return sm_reviews + return sm_reviews.sort_values(by='timestamp', ascending=False).reset_index(drop=True) @app.route('/data/person_sentiment') def vis_overall_sentiment(): monyear_percent = get_overall_sentiment() chart = alt.Chart(monyear_percent).mark_bar().encode( + x=alt.X('workshop_name:N', axis=alt.Axis(title='Date'), sort=['timestamp:T']), y=alt.Y('value:Q', axis=alt.Axis(title='Percentage (%)')), - x=alt.X('workshop_name:N', axis=alt.Axis(title='Workshop Name')), - color=alt.Color('score', scale=alt.Scale(domain=domain, range=colors), legend=alt.Legend(title="Sentiment")), - order="timestamp:T", - tooltip=[alt.Tooltip('value:Q', title="Percentage"), alt.Tooltip('score:N', title="Sentiment")] + color=alt.Color('sentiment', scale=alt.Scale(domain=domain, range=colors), legend=alt.Legend(title="Sentiment")), + tooltip=[alt.Tooltip('timestamp:T', title="Workshop Name"), alt.Tooltip('value:Q', title="Percentage"), alt.Tooltip('sentiment:N', title="Sentiment")] ).properties( width=800, height=300 ).configure_axis(grid=False) @@ -641,9 +615,7 @@ def gettimenow(): Response.workshop_id.in_(w.id for w in workshops), Response.comments != '').join( Workshop, isouter=True).order_by( Workshop.workshop_start.desc()).paginate( - per_page=30, page=1, error_out=True) - - # comments = pd.read_sql_query("SELECT workshop_id, comments FROM response", conn) + per_page=100, page=1, error_out=True) monyear_percent = get_overall_sentiment() diff --git a/app/forms.py b/app/forms.py index 3291bbe..2b8384b 100644 --- a/app/forms.py +++ b/app/forms.py @@ -3,6 +3,7 @@ from wtforms.fields.html5 import IntegerRangeField from wtforms.validators import ValidationError, DataRequired, Email, EqualTo from app.users import User +from joblib import load class LoginForm(FlaskForm): email = StringField('Work Email', validators=[DataRequired(), Email()]) diff --git a/app/models.py b/app/models.py index a2da9a5..f86ab72 100644 --- a/app/models.py +++ b/app/models.py @@ -69,3 +69,4 @@ class Response(db.Model): venue_score = db.Column(db.Integer) satisfaction_score = db.Column(db.Integer) comments = db.Column(db.Text) + sentiment = db.Column(db.String(10), nullable=True) diff --git a/app/routes.py b/app/routes.py index 985493b..5b42d30 100644 --- a/app/routes.py +++ b/app/routes.py @@ -6,6 +6,7 @@ from app.models import Employee, Workshop, Response from app.forms import LoginForm, RegistrationForm, SurveyForm, ResetPasswordRequestForm, ResetPasswordForm from app.email import send_pw_reset_email +from app.sentiment import preprocess_comments, predict_sentiment from datetime import datetime from sqlalchemy import func @@ -142,7 +143,8 @@ def rate(workshop_id): timeliness=form.time.data, venue_score=form.venue.data, satisfaction_score=form.satisfaction.data, - comments=form.comments.data + comments=preprocess_comments(form.comments.data), + sentiment=predict_sentiment(preprocess_comments(form.comments.data)) ) db.session.add(response) db.session.commit() diff --git a/app/sentiment.py b/app/sentiment.py new file mode 100644 index 0000000..ed8cea2 --- /dev/null +++ b/app/sentiment.py @@ -0,0 +1,18 @@ +from joblib import load +from pathlib import Path +import string, re + +def preprocess_comments(sentence): + sentence = sentence.lower() + sentence = sentence.translate(str.maketrans("","", string.punctuation)) + sentence = sentence.translate(str.maketrans("","", string.digits)) + sentence = re.sub(' +', ' ',sentence).strip() + if sentence == "": sentence = '-' + + return sentence + +def predict_sentiment(sentence): + my_model = load(str(Path().absolute())+'/model/sentiment/model.joblib') + word_vector = load(str(Path().absolute())+'/model/sentiment/vector.joblib') + + return str(my_model.predict(word_vector.transform([sentence]))[0]) \ No newline at end of file diff --git a/app/templates/accomplishment.html b/app/templates/accomplishment.html index b7a1161..8c4c832 100644 --- a/app/templates/accomplishment.html +++ b/app/templates/accomplishment.html @@ -157,7 +157,6 @@

rate_review Most Recent Reviews

{% endif %} {% endfor %} - diff --git a/app/templates/sub/single_response.html b/app/templates/sub/single_response.html index 296aab8..23af1e0 100644 --- a/app/templates/sub/single_response.html +++ b/app/templates/sub/single_response.html @@ -48,7 +48,7 @@
{{ review['sentiment'] }}
{% endif %} {% for comments in personstats['qualitative'].items %} - {% if (comments['workshop_id'] == review['workshop_id'] and comments['comments'] != '')%} + {% if (comments['workshop_id'] == review['workshop_id'] and comments['comments'] != '-')%}

{{comments['comments']}}