From f47f459b9c6cfc65c5650f918f9b34ade02776cb Mon Sep 17 00:00:00 2001 From: maxalmina Date: Wed, 17 Jul 2019 11:01:05 +0700 Subject: [PATCH 1/3] improve accomplishment performance --- .DS_Store | Bin 0 -> 6148 bytes .gitignore | 2 + .vscode/settings.json | 2 +- __pycache__/config.cpython-37.pyc | Bin 1657 -> 1631 bytes app/__pycache__/analytics.cpython-37.pyc | Bin 21138 -> 19799 bytes app/__pycache__/forms.cpython-37.pyc | Bin 2954 -> 2989 bytes app/__pycache__/models.cpython-37.pyc | Bin 2770 -> 2813 bytes app/__pycache__/routes.cpython-37.pyc | Bin 5734 -> 5860 bytes app/analytics.py | 72 +++++------------- app/forms.py | 1 + app/models.py | 1 + app/routes.py | 4 +- app/sentiment.py | 18 +++++ app/templates/accomplishment.html | 1 - app/templates/sub/single_response.html | 2 +- config.py | 2 +- environment.yml | 12 ++- migrations/__pycache__/env.cpython-37.pyc | Bin 2204 -> 2268 bytes migrations/env.py | 43 ++++++----- .../versions/429782854307_updated_models.py | 61 --------------- ...429782854307_updated_models.cpython-37.pyc | Bin 1759 -> 0 bytes ...be21e8059_initial_db_models.cpython-37.pyc | Bin 2395 -> 0 bytes ...y => b7a1e1a27088_add_sentiment_column.py} | 20 ++--- model/.DS_Store | Bin 0 -> 6148 bytes model/sentiment/model.joblib | Bin 1142863 -> 1143630 bytes model/sentiment/vector.joblib | Bin 7551123 -> 7551163 bytes requirements.txt | 40 ++++------ sm.py | 28 +++++++ test.db | Bin 176128 -> 294912 bytes 29 files changed, 139 insertions(+), 170 deletions(-) create mode 100644 .DS_Store create mode 100644 app/sentiment.py delete mode 100644 migrations/versions/429782854307_updated_models.py delete mode 100644 migrations/versions/__pycache__/429782854307_updated_models.cpython-37.pyc delete mode 100644 migrations/versions/__pycache__/80dbe21e8059_initial_db_models.cpython-37.pyc rename migrations/versions/{80dbe21e8059_initial_db_models.py => b7a1e1a27088_add_sentiment_column.py} (84%) create mode 100644 model/.DS_Store create mode 100644 sm.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..80d54f7ff58f85dafbcbb66a0460e733d0e513f7 GIT binary patch literal 6148 zcmeHKyKWOf6g@-MB8o}mG6ltc0E*<6hK=QX0P+VA$Bs6M?Cc7ML`W1{q)LT^=&4DS zGM)JhB#JyFYJLDI=gv&b&g^aw4MJ$HHFI|7-aGf~*~fSQsNS1#7w7<3W)qB7*wvW$ zrB&>a9eGMrGRGwC4o2BDuSAA{pg>UIUsOQe-3S{kXFV<- zJb!th^;&q#I{99wUh9YG6Y(w$afG2$QC)m2`@EXh=jI+Qa^0&&#qZQrbE&(6DWA?1 zuXXi(J}rlsIUMlfwkgizb7SvG?VKLB2V)(ccrQF# znh2-<*<9-Xsd3E~TW%W+eNZ4M5ENJ_Am4|WO)#-o7}TE*c6tjyj5w@~$Fi4@oY-Pw zu`oyvO_@}pNmX`>p-ei*V;h%PEDV}-D7*Pkwz9H26s4f~OiNcfVb770(iS8HrnkWh9K@-kM_ib=xQ2|$x_Q>#y;9XG# zU!D4rAbXluODK7Kn-EuT6kvF2oQAQoL8%hLL6nnsqn&A@Y$j#8OL%fjJ>WAv=(w1! zB5H_vKx*!EJNte0SAp57T1VvpHXtGjV1yFoL7RHgGuOKRS~c)J_M6$L1s7Wokq%%) zq4ylMn7o?q-4YnJjHm|P>Ga@AP$a#NGhC{W(UJCl!G{> sDAWp8tRgr9x1Ua$wNlEqFNWgkT(*;5X2G>|N_3Ke&Ez^oB#i3=FG`Q?lY&ibUmBE?{vvJ+8*jP_C8#AfIjfcoX~|K#s}Qd z{VK4_9HhYzW{-ge8OVzKAnE9$AgnX)jJFe_C`!)>QR;EWL{}CSbQLgrijzlQ6*cs= zkuM0gpRnsMtY6-;S)BLI!Mi#R+#5I!&n39xwa?1DZsJ(6a^Z(Ek4jpEBB2z$DQl?) z$wu_0>@g$SQElX%df2K^TqV>9lZZIEvDrFk%by#Rj+9NRiCD%8RD{MIdgz!4gg-Le{<)%-+g_r``tS)eujPXI;%QfSy`swZ|3QpkNntpp{j{J_`(h$1)!1NEKci@EMvYO6XS=b{ zsKc|vs5ct$>@?fK)n#^=U1-y7cFJdu(P%WKRc)KmY;3}_*Vt^dfID6^wivA_Y&W(V zZFuf5gNuDe`w}xcj7~Z1PD$xnVscX5Mvt7-HlvqnF+{(tkFVBmm-Rc0zUBG>S-*3& zzF*c47`tSBk!iam!{92zZpkoY>{;$$P}UD)N+ZTzqG`J&WgjT}jRTT0Bq;|$Ibp#!ZX>z{>HER+b;f`V_iYff$p9&1&cLG1>G{9A+nZaDuP>sN~z*WT!Oa~T& zOY{sGK_hgHH7Qrwg0i4YhZopPs9Cv+@`7T7n-wE66Y$?L>*$1@VTy8b?CjWyOKseA z@^;QK`)9@9Maskv!-cxOHKJqYTq_F@k4GBClaY{kE)o|vBIiXgdVE(Mrge*!iDiXm zO>@TaqA5E|{LXu9XFS5|QOO$!A^@=%-OReg8_|}blC`74t4Nz}Vr443KyfMn3JV4* zSOcq4s~PVXZ&uWGN^vxy3JYT-3R=1mPGgaa%94WRMa1#gj)5+)@NEDuwz>w)GMKXS z^Kr)esJdhh;)Pgam~y!^HuVy;?_iEoV|sxt5(vI4PT(d=M3!#z0>~ z{$qWY)U?`P)@93_brOc|>PJ9b8DfMuqB%2-?k}5KG;cc=m9E4(_ZrEhu{$NoH`H$k zfGvohT=^bFyxHAToDLc)v`RWv7gQ`dIB|(XrwZQ;h=+J59$p}0r+5odX^LLZ!h1ZR z1z><)0JHJJC?zbTR1m~iMzhZAT+cf9;@yb?6c%PJPy|1OvZ$_Z7|EmHDo?}8T;{TA zbwQcIoE8I1utGPu7+Qd}HY-3-ftYJLu)xG))jfSQ>FF5shgf$_osJk4M&&hzWmHW_ zp>nbSPQvHA#V4yTZ1dRM`bkC)W%O*)&`0kF+kfDvT-hdGtEm^=HQS5han)0k^IlAM zQfAJua_LK?e1qHhc$r)R-%Y(sHIi_op=!|-w;jjTlU_*A=QHzOIZP6=uyTeu=LNH7 z#`Ve_o4bk2dZu7H@c<{A(ZYF?J9f_7khOF3rq1gUAYfp(bkgG0noT1g0Q;&uYt`e7 zp9dvHs-wR!eydKef+_{f#Q)Z8DMG*tBw)Dw)xZKd{j%*}3or%mVaPKAUx?DX8*wAk zQG@}sxEfpt8dBP6C}9z5G0M{F5?hQdDT~m+61y5&2u+uxZN)W)zCxt9#fpW%Vx_?l zC@Y(lB~}DW6*ep7!`PZ8)kAl(q4OnH!WFnH7bbKYm+Eu*%9s8YlERe=8ov}*IR!s2 zQnYgUjGi)i8(!l9FYFYO(2W4A=LtG4XJH7uzkKs;?Wc7ZO#4Y;{N6% zY+QW2d4G*o?pkifJeEsM*&G4lVet>mo2niL^OcA+>;v~{BD$%+_$WxQ?gVx77?neE z<%Mj{A5y1^BtZ&bJ&I_6*%&B~quXcj^vdK=CrDg;5`=gQZ^y$A_hE!EsDa0cvYp@z zzzge+I}g(u%X@V($7t88=CZTggx@me3 zV+NSE7d~z3hRJ!BIA;iQ1U3ODa0pxg*1hcjy%Y#a_F=qw6?zKeCej%@sb`)M4{koj zUKH0ilfqu#`~%ibO65-gAdXX1{wTGIS~++bB+QSAuW#8}uf?)MeMwuvfO#T9ECnDzR=oVe2=x!7UY_9h*@c_Qx-zn#e=%>^{Zl!LY5%4 zOliPyawr99fQP1ctJNRH#6z-x{+mQ%TDjwLd&YdgGTf>7rZp>&F^vRt+E*Y&s*EW! znNnS5teh!z>FbVs#|vKDTwOsD(26JT(10&4g6zMJ}i890I0R zh3BM|E^sqJan4tFIVWvA>BwiP8WL|BU41g$!bEr~n)0UMOSF zVzgCbY4uHpf}dAYYAp%s^cC@Z+bQ6!~Q}dl& zR#8}?K0ZwFEcFpwvqdlHfSo_`i=bP5Fih(x0M<&OgOly7#1DU5893_JOa^Ba7yF}4 zIgDi+{Fl+f3&6f){8$bH;BM++Chh`hhhG$bcD-u0i2h!+hdBf;+lJw<7V4M85yu?QA8@rHWXdZCYel)q>|>yT?}WVJZi|S zXYHurOa8ZLF|=X$VPZ}46khmm6G$D%_~Gv<9HEJSmTE?Sk8o|n2zwDk#)Qo+8A|vK zs(zi|3k0j7VOfE~?rmaW@m%i^`-XU<_xn3)iR()QZxGB8tm+Ml8YjY21kZ`Tji0^C z?##)dN{o!8CoOFp0wX87^&VhZ(DAyv%H=E6=1&O7tN5Q1oF!P@T)A(hl-%8QAC(dz ziTQyw!2g=c@9SCBaAKTH5~+;tI51@1A^vCk4)ty2#`Yagu~rI$SiBSti3GEsr~a-G z{EU&3q6{Ksd}q(81!^ukT@^8hL18Cd35RV{c7Z$mNf1kgznUWP=TIZqh6eKdJ2R#C z)dhcxdR$49e7(ds-&GgC1kQEIk*^hvDkvx7*A)1{T0SHbjrIDHTi}{)ON)NrwCDd* z*UFjTvLn;0156RG^zT*QQN+9boydP81OFH(*7!@@NHZj*X^{tT^+KuMpAqRnf>D4M zo5|U;8PiCcWTK~uzMPVbQFX|rv<5X~`^c@4^UgpS*NN{F1Ok9+ieXp6IX!D4cW1om zF}!=RjFp~pDUZ2z$Y;ArVJo^4w#27~I?LrIMXtejiLVcB?R!o?yPNC zO|=r)LdLbY42Zu&_0m9SCVqTur_!GjtQnFFlG2)9qY83)uUzK1bP*lsT*6ErP=DB=Z@_8c-cSParLrh5#T9d%FHRLLG%XXTJM6jH9u6J<#p z7-kH6HusQN8p*2fDmSY4YHCY2&4}?HJZ@D|bFxwX#{{+F+5Hm(B^eqr=yG?8niyt;|ZMPJLbChp8LY3*%r9@8UN_#la5ts2Didv_}hLhoXxg8p;dX zlasg^fWb;Le2m)NY5w9v2e-B_5q}g16I15mzlK-->%_0hgcwJaFCE+$lKK!oI@n%3 zMhrIrE4z<%Qk9D{s8QEiI#&|nc2Jvr1QeTb67&s8%IXY}Dd-^(*aa+r?AVLoD2|hz z!@nyV(1LS1SvSI>+}Wi~-OXsFnU_i=)VwC6EW4grk+h?a1JHQlgEKS6h{sx$`q*Ddh=zNvv~RN z%|VJp{)MUBp(u=U+6?9Wsc(EF7CIQ7BJyc*eYCe~bvMJ9KUT!sqdyma6184zWm|K! z3dcVs%Kc4?;Wj+4=a86bV&X_+e>?Scl$wN`DLrrEiq^!rxsbCSFPMJ#mii$@@N+^O z87wdHDDl>jdi5tveEUdGs1obZZqytt3v46L;oSuPOn^1Q$mt+C@2vNwzT14smzx}b zgeQGyE3Am0z}uBh(K^1R-@jw!-==zLC{hOhO0WM0fP8LI`8x#PRYc@u`bg;*DwhhB z3c7uuJ`Cgp(Ts9w+~*EdG zT+8nj_Uu0K?AW9Ia!ffPTnP8?@|d-*$$MpoGM3|#o<@&|+VO|Cmo(u`G)D8C195z~ zQ;v$4#)p4;_7guF`$=*CsVSz4FQ2OK#9f9yml(++5HE@QbC zru2R!i|1s5?wmGL3w#q*_!fdzfF_RsgS)f55pUjxwao>0_P9Cl_1sxvaxx~=(5D-m zyc2p(&&<14%BdIHJrA+Y8=t@Dhm36&uikrVKw48BjZ{z20DzO<#m3~=7e&X}gY8mc zPm&BmemW}&CrL6y5vwPz*T~nngd2DvQ`Na<5uGE~%^HE*={aLHKgqwHMOjgZN z1~}dWT2f@rojCr}aGK;~_~8SRyrZD6AQ8*43%DxdhsD328|si7?V4v(U|e%HS_Knt zjh<=Z^fe-0rA4Q4O9NGz9A6m}-X~t3Xs^>EGL@Xg=fBB9CIfA7O?+#jr8q$yJU|d6 z?l$V9g9nJvO0X5+MQx5-!$fl4-7)?8EF1-8T&5}cXEh2J{6_=>)Gz=~@B+9J z_iz6HhAN~Awoz?x+7@%?D~soeEaOv{=I{``wiC!j@1v5m|D#koM-U~x7=g6dN{LhYw8uTOo|;1D(oEg96Lb>v6YL=v0mvI+`DxhyRwmW> z2-W5Z4iQM%l0>}_KCICFD^Xu0_#(kq3BIO47W^Aje4F4Mf_DjMQ|7dj@lFDgMTxyj z%a4rK;r3LP@M4$w~cqtvA>4UbYX1^WBG zyIQS;XamydyZ3(QJLjJBeZTK<&ehkR)86%hR)4&%E*`+o=g)6{q-W-t`pw$o&+K@v z6wS6yw3-1kmTk*znb@KQa9@*c&vi_6XaOrcv(=273A6U5Zq}LgPX=yi6Wgq9Hv$u# z<_5Fj$-qRHxzTLIwcBhmn{n+i?=V|%?KL-*1K5?hnVQ$5Br@77SME6~0m)VW$U1pEji|c?DS{yXDFKK3< zxr3**n>qbU8c#ZD-pP~RY3`y{j6B5ccU9U4xV+^3rWW zKr*Oer*eg?GiO=y-Fi_|w-UkkX_3un6Px9SLV2xCZVrEJ*A_5Tc*+*8r*v~lG*V|x z#jJ8?;rF0!t@71yx7I5=BCVwY7+zpTGc`{)bu)M~cs;NXFhi|@#h@2j(iQ?UVW2GV zX59=;=|`w7%x#fXZDDHjqBGIO*b-f9%!nDiscjBi#|RiBzM##bAB~Nr#m0>{YJtF| zQx}4#PF$AHMDLIgmt^kUv2oRWzD+DglM64ZXyz;&Jn})<_eYyx}H$MtdoYOvX0@Ym}z*H zXXntAcdn`$x;3q=XNc`IqONL$B_E14m8!Hy1J-6NFC{Fu;N)FvH#lxRNI(k(YxQ=m zU2oCqHL(|*G}g&KbdBKe1pZu_9yF5*6jHXS>i9u^ zFV@maQxbh3uxlvo6y}ne7$RQPFOW>T^y8`YlchvNNfj@{X}?inmB;_ zb;bZ!HTVm>Vp~_;6qIBf19y4ML%6RjEQp87!s1w_7w`b48vxJ}oW|5P1#XnI>)IN& z4z_k(hZcgZfte6%LHEKl5k^i>Bh*BF6wy6NEym24S@UFokylejWw4A&zdgQwd#N%& zVts*1d+qw|W(d<+eq|lMQd^8K=!*%z=aRN+QoaP4vwnQOV=#Bw0jLIDz!&(+dxJT8 z9++_^v-aftR{yc8`HGeH98qp(i>rVNjXQbkGE5(}^M$ODw&YS%%h?#Tzp#uQZ5++v ztaH^8xY4dv0|Rm@b|h=$CQW1X9z6XJe}Va)f^gE7>&jOW?WKxM39RHem+)5x>msbW zr~|T^7n}(#1(!6eP9GN|5J`sNIl{=hS$LD!3684C8M7AiR0K|E=QAo%DCW~%5zg!6 zfx1zMt)5^hAdZ50ff~j@e)H)FLMu;19H%>SIu$YPjP1GWzM9`$3fxP=V<4+u9R`Cn zK{3?}EJ6n+fV&i2)Y)Z&k=Qj9&0F=omK+Oic&cyB9>v#|_{RHf%Buo(yetwn3m+q7D()Bg=NYTdec z0PU-E76OAq9qA0dQx0}Fb#i7PCp3PW0tfIcj^GNoMyBq!v)Q~s2p1Lyp5=)x&nu4g zm7C~=F1;k6)$3vcR~5@!8G!4GrD6rgwdqbi*59%zO;Yqj;+1)g%boGPB?O%CVgxBG zj9`O6%qc6lpf4b-a4^0s$QMLAk$w=&H%XV6=p`1ZU33tkSgJxca*G(C=RqQr8ImCt zL=sj(L{SyN9iqC}L#>+wcQFNFwZ|#u77StJ+$prXTmotltsTjvMHhzrV!Uud!fbf0{pZ!017TBLSqJNk{yY@7Az`0&hcLi(;PT4 zoR?1GzHRGxr0l_kpJpNRL>Loj0v( zDwIP~O$NoIBu$G~k%A>$C%@LjcaRR8E0!>_*%kQZ(ZfZOfj+gk)~Jgd7!|xxq#_?> zyw$>9W&I$%$XH!RDToq;(nARP2`z+B&~VZzTC6uA zS2Al`DQ87yHk5N#EhmsQVq~UOQhpLyD}@QHlG6RLVAuL_wx$YSF|tKVoQFnEUq%U2 zsxjQOl{XQC+1ErDS`Zf^2ve|-3e8$p!EfOTfv{J0OCKX->|_HQUPUv)DHbN@#OuIR z!JKi;M}ER}gqPwsyJ`c)rj%JMWbL%!l`L001nHI1gWzD~g-Ohr@;C*GQ)RfYTP34v zL(a+1S%ydfD_DV71)unookgyJFI`h{Ba2#$BZjK?VO4pKySa01%z8ngy|Gu`QZ8h&;0CV6>7f1G@i@%BFX)P~fknSdYG0Y|kcp>@8m172{S z(arjs;hB2cf|(m{>HtBb*>n>H(So+v!1<$Db~W^A8F{v$w?tNh%_63gGwi$yBdcaC zRdZ>%bU+)@g^{&XBx_}0)%gU~6mYv-VdS!k-{)(}$XN4HPAenh$j;gvJGTm_ovb5N z*b$~BRMc~vtY;VOPK0867aS)kzuC~P<>Y@i9J%9h%tS@FgR3GJ_+?6-+qiqv9JPeg z(}wV>RVv@PvF+qr(B?LS1oVhruQlVhQ`5ChJ+8%c@d5DX8waVL95mADVlI`*I+I4$ ze*At7>8s3=&W4s-dZVprO)}dwq%BHgU$gvHd&i#V(O*UGE23^n^Q-@l{Q4qf5w*1D z0SvRd`KUH63(bc%D(K70TF3I~X-6O;mE_@yA;zgi7x0n;N#JNf2?ZigmC1z8GfZM#2 z#qfH}Sw+rT28=EwNj^j5St9Qy@_r)E5&2yZEIJhk^ajN55vOWzfk#zaZMH=@+;VK| zC&8cJ^vgy~W$nDBZav%bBdxQEh9Zj~yX%M^AE&Wmb{;MarHfJdq0QUwz}%gpYfuYT^r0wholUX?{{6&$@=dm=Ol;vB=0}p!sIjRfr(W#-6X~ zIWRoAHx0c&hURm7E6MBPZMd)q_+REBR02hzGiyC)o8EM?Z3PU%BiWCW^Rp=_7G|*t zK&4!5RZ=Kh)wkliAg+SKnqqsZI9tBpK&v7z-`aMnguQ?<2b=!|jY~#W(JN^ukC?3L zahcjDjw28W_yw-`2bL7YtmUe35l|L_ww=|W z;i9RP#ZIkV-A_T5__*%Hzk2^~tiS4DGqZ#p$6M8)F=pQkbZfCvGxY=t$T zZ+R_i`9%UO^7ddZ^rYC`H4y*A zL2xu#i3=fGrb;kI(zZw=mQZ}CLQ;q(F8-33pCIB#1+HUP!on&8lN8&xhPHOoY3yo$ zDu1%Ir|lKMeB-*~PZTRSgtBtfq8x7@f;FhZBf z$@Kp8q_v*`#t3C(Gs2qI+uryF`3fzMQjH&0Rt!SnNPLY5n}*<1&PD>EOnie{{)xy+ zrirL10%^(-_9%xQ@iMjlGm*8ygm`S%rgDW>c2!Pw-7?*?SNny0Z_f{Qa;mva;{Qry zkx0e3u;w%|K1Adr@}s>M*2(kija6_Yg>sTV4dsz~ZodkG`={yBysoN#Kz)8l zK^$;X)%YV~T_KVte$6b7ud|k!u}B*Z@ymq`#SInq9M8z2g}{%}9p&>ZEb%##`#T~O z?$G>cT*?U%M~PxT)xD}FYiFiCs)cSJ@x@*tv0okB&%K|1ZXXrw)w=is7vB`X#YcQG&IA2xJ}UT? zdCg#s!x8qC7v#3#Q;{!Ib4kt&_tg?S@tLD)VoqDRt3S`H0UlO6u0qt%m)!GiUXt|HcktQSO9@oMR5>- zZKR7b0=-^OT?yCX?d<1_ruERu08fx01N`fB$9R7?-MyE{H;FK+zeRT(L^+%?I-`DF zu!owy1ETc8kkY*&6{4*Na0bLIZw-2AY6Jk)=vhWCmBo&iFvlB?ugZ>tsrXTd*q=cn zQu3*Te=|fu$v@!b)U`;D&lBOT3&W0dLOadBCblE59qO;Iq&Yz2uqHoz=(qA15rJ|! z-?^&R^@=8}&xudf;M--v$fJ-Wr(Jde?4>-X;92=(oo^(I1_`^<27Rf_TG+D}^Y&vn z%455*xyUZUlaGxIb(1HoU|u!Q2T>}sr=6Tt!1>P>U4C|?FMJlpAqEQwwkndi$|+D?ys zAkg3ry6-2F)MVG_X8E(xi$|;9qx;9=>%8nNUgnZtQRNQb3iGKlDgSLp?`~rk<}?`Q zZ?{$EddKNwvS+NT^Fey;8jaC$JAaRJu3nbr*c%5b?eKDaV7JojHC#1P`I_JOsN8pa`p|K_JDE@s z9Mq<+PUC9l>55aLYpMM?giYLn<=^52;abYHr>0VQCBJgKAIbRV$6HI?m^PnT{mKX; z4$-pN?3M5#Mu^85@G#x21dbMrr)o~(1m!foby$KyLf{uJwFEIC@4=@aGgrgIa!K&JW9kOG6mxAK8E(> z#2e^oLzQoOhUaILq+VME%U0QSvPBz`2T#61^KLzR^2eIiE}uSqW{?5)dK&m4A`>8p z4(q^vY2J1Z} zcYz3lwN7b8f7vDCJx=5aA`C!1Bg6-3%poE-h~M^RqO63AbXR3e`Tg@-aHyTQXIKl# zWA|*2yaNKsyJhyCvyZ?F@J}qZ5OErb60nN+|8C%aV;&$jC2-Vj28`kQpYX!@Vy-YJ z?xoff+*%Y_{Evb-%vU)YY&u=_$wsf{FA1HE*&m&idPhlo5tWCFymDCvJ? z{4a-W&&R3tJwyy5tXtA3K6)^`K0w@068SEXpAh*ekzWwOhc;AlI?*m7l-UI(JK+%_ zT~sNH)z*rB_#X&-07vOC3W0$8C(zJbU~C!kg%l@-6bT&XcN@eD(k54sl0wpSUYa#7mk7#LLnn z?r9#%$_FE=4({(n1Y{xfeT7RcnQp?Pn@foH)P5g@3BAQ+0_V z<7@KJxK4J5>blULW49wfc7BrSZ|pGs>wN8%_Aj7K$Zp^(hW`(-K`kp38Mg^~^(G*% zZAWx*`z#(?U;3T2+d+>Ok^d5HDY7BRH_+Xa9%U>xdwqG3CU1e-G9K4~MQ|XXaB`mI IIiBm85BBia`~zuHrm^#!*+K<%;C#$C^UXKkocR`<2Hw2q>SEKh~Qno^21_UljCU*LbAvv1(ZB}3wP(3n2wH%2z0{w)Sz zjLwJpzgPrce{_bPSWwP)CZ|nue9>&0cE_m_eh!DjY5mh_iEg4bK%ZN86c9O-%hB#| zJvta(Y$VCS4yt-BkB>S-nodvnnaD@5$S}(AdWad4Oj0E*6ZC&oKwhiYde)t-SYPDN tmXN>M^b)xN$ia>1xt`yO+18HQi=z)fvVBjPMZ{(ZE!MJAf0*>iYsVrHnsVobGQ`i?WrZAdP$VeiIiyDeS=vnjDh@Idz!*G=(SEvTIIW a$5}Jkic6MT45*%ik%du!QDSlm*GvG4J`jxn diff --git a/app/__pycache__/routes.cpython-37.pyc b/app/__pycache__/routes.cpython-37.pyc index 63dc72e2089dc58d78f157d5bbeb85d70f3ec023..e5add6ebadc9e20c0b03673707318b0919ed8730 100644 GIT binary patch delta 1987 zcmZuxOHUhD6rMYN;CC=Kj{w1d0W&;oAP=QLC{5BVsw#<^L=@+NroIy}u|3YY;}9Yu z7DTGL$RfR)N|mCptQ1vMv#8W%H&q|`1GA{qC`wmV*R9m5=Ufx)pjh%Z=brDJd%kn$ z%zZWSax!*08Vzaq`}51}^-nt=#k$ywN13m;2l4~u!Tcc8xW+@JbUB?*D?3~oDi7y} znP!F8M|gxsA80!)KVpTSWO!_sspp|x##?!u801EkuMN2nPw?cP*HGSeDA{4noUTaRO*4DaE+{K%@#`yT2%#rqXAp@Ijn*dR|U&!mbSYKk4^BZ|o?Cey@> zazimW#f&vE<9y=59CbCN3QQ`ES)QYC^iHdUsiuT!KBKmp;YTlN#<6Gru_b2cZlVfS z73_i~#7e;~m#vB;+&DUUvEZx-To%bEFMFOOQ}_m4-_0Tj=PM0{aRtLN;N|fsn!=Bu z>3-(dAtUd5O~X(D`!Ir*>nm2cbvKEo@FHj)KZ7yZ;`^43$?tu~Sik(smz$kHW^aG8 zRA$U9puTFUDAq_&HxeR(pm{?qz#{mMG04e}{U40gl~Ja$RC>U=TP@jODVNYG(F0TR zSATVKdOxxe9yBXvY11hd#2B)5t=JpP$YRTn0rD9PQu1WrA{&sO1tvNMX_GX8IwdbR z9TSG+FM;{gN%Xqjny}zNPjCX6oE!+qIKQa&^+1req2mD`!Q0yLm+V!$<}4!L z5O+%c5PaMaw-#DlrV96>V67H~V?lP!DVI<9}6gw^L@E0s41CguPDUZyKD+N*LqTEKc zHxE(O#WVaVl4^fzCHZfphh3J3qdn}bJRW_DGxQ>wHdN0Eu$l$G(Ru4$=Q0~RzOAY+ zkd8S5>J@0N;9Y|E2s{KdGKRd^VoELG86Jv-&HjRU9pPjG(Ks^A>M!lt~|y0$>UZlG$eS}T=W*=+&q zPSM&BMjQB%fqnv|z43Yzu<{WO=JB<4t zhVFZv?)9E0y<6U0#x!Lg+11eMfS*@R59;=xe`1el;IzX6O=wtDh&&9`7NGN<%L}V|o^8sYZyV2d)uZM=*S-*G_gB zG(m3Qwa9~K?gi7a?qi6^%5RdHwh&9|G(!O$K0)t!2`FuMcHLKO^e$$}jC?%?i zkhonkONBM7ya|1n1{o^q1H3*gpc@Br1XBbv1oZvDDT31kX9>;`EE7=WCZWSxC$8%s Z{e0QxwUV{KFhm@a(;W%cD$jJJ{|B;~w_N}L delta 1817 zcmZuxOK%%h6rO939lvYGj^iYLCCURI&7i4eBpog(|^0*CDp7+L~|fJip!AJnooB@HfM99JzzwZZknFfL6ML9sg-WW~FV?^#y-?U-jtp*d$3 zr%`hrpH|W{P~O=w>dji!tkm^}HG~S2@+zu_xtI^10E2?~+t{qv`9aBSXn=76TX2sWizq+z0(-#~K zSu=Kxed`_6cgQV?m+nU$a!))f*D1q2Kj^zv)6$_>vFbG>PK>k=^ilK_K_^+1wr>)I z2ej7h*)$If;BZ+id2<1 z-y)X`{17AGPPQ~{FN!Gh!WhYh3G4(Nrqn~tCeQLp4pH%)H=5W$E3+SgM$^TjOhxqA zXkuVh82)c?@Zy0C(pm|O_$962d0|8)@Ww4ku7vJznSe?GGhu&Q0D5;Fx5dlAC|i4$yu=t1Ne9=nc?_x9TDIbBiyidT^&fX>S8w1X!}6i{ zB3NO0@ozARmnIpCIn%^tRFpzD7bph2kFe#&Zrw2}M$?3K686yX-rX;{cbvLER;}3H(Bt7_^^7q^u1C%k zT*bNbKHH#o-?9Ce?IrL@a@VPf9}@REPF`Nv!;9?^b;E|Z627f|fO*7E;S^gCt?=Y# z3h3(qX#zP4asWnf+j8z(I}PIiuF(c>*pb;g6NBVMEpInUHZMZk>tZvqmZMZ%-CL%u zY$0_Iwg_$@XbzXC=$#P(7lBvf)#>Qf;ECof^C8h l8@X!VyHefY`!#);p@li8C_yF8Vl0AxA3kyB6F)~Y{{h$fcY6Q; diff --git a/app/analytics.py b/app/analytics.py index 10ed43f..2b6e1e1 100644 --- a/app/analytics.py +++ b/app/analytics.py @@ -9,7 +9,7 @@ import altair as alt import pandas as pd import numpy as np -import string, urllib, re, pickle +import string, urllib, re from sklearn.feature_extraction.text import TfidfVectorizer from pathlib import Path @@ -46,55 +46,28 @@ def getuserdb(): # =================================================== def get_response(): - responses = pd.read_sql_query("SELECT response.workshop_id, response.satisfaction_score, response.comments, e.id as employee_id, w.workshop_name, w.workshop_start as timestamp\ - FROM response\ - LEFT JOIN workshop w ON w.id = response.workshop_id\ - LEFT JOIN employee e ON e.id = w.workshop_instructor", conn, parse_dates='timestamp') - - responses.loc[:,'comments'] = responses['comments'].astype(str) - responses.loc[:,'satisfaction_score'] = responses['satisfaction_score'].astype(float) - + responses = pd.read_sql_query("SELECT response.workshop_id, response.satisfaction_score, response.comments, e.id as employee_id, w.workshop_name, w.workshop_start as timestamp, response.sentiment\ + FROM response\ + LEFT JOIN workshop w ON w.id = response.workshop_id\ + LEFT JOIN employee e ON e.id = w.workshop_instructor", conn, parse_dates='timestamp') sixmonths = datetime.datetime.now() - datetime.timedelta(weeks=26) responses = responses[responses.timestamp >= sixmonths] + + responses = responses[responses['comments']!='-'] return responses def get_sentiment_data(responses): - return responses[['workshop_id','workshop_name','employee_id','timestamp','comments']].copy() + return responses[['workshop_id','workshop_name','employee_id','timestamp','comments','sentiment']].copy() def get_reviews_data(responses): - return responses[['workshop_id', 'satisfaction_score', 'employee_id', 'timestamp']].copy() - -def process_sentiment(sentiment): - my_model = load(str(Path().absolute())+'/model/sentiment/model.joblib') - word_vector = load(str(Path().absolute())+'/model/sentiment/vector.joblib') - - sentiment['comments'].replace(['','None'], np.nan, inplace=True) - sentiment.dropna(inplace=True) - sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: x.lower()) - sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: x.translate(str.maketrans("","", string.punctuation))) - sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: x.translate(str.maketrans("","", string.digits))) - sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: re.sub(' +', ' ',x).strip()) - # word_vector.fit(pd.read_csv(str(Path().absolute())+'/model/sentiment/train.csv')) - - sentiment.loc[:,'score'] = '' - sentiment.loc[:,'score'] = my_model.predict(word_vector.transform(sentiment['comments'])) - - return sentiment - -def prereviews(reviews): - reviews.fillna(3, inplace=True) - - return reviews + return responses[['workshop_id','satisfaction_score','employee_id','timestamp']].copy() response = get_response() sentiment = get_sentiment_data(response) -sentiment = process_sentiment(sentiment) - reviews = get_reviews_data(response) -reviews = prereviews(reviews) domain = ['negative', 'positive'] colors = ['#7dbbd2cc', '#bbc6cbe6'] @@ -103,31 +76,31 @@ def get_params(): emp = getuserdb() dat = emp.loc[emp.this_user == True,:].copy() emp_id = dat.iloc[0]['workshop_instructor'] - sixmonths = datetime.datetime.now() - datetime.timedelta(weeks=26) - return emp_id, sixmonths + return emp_id def get_overall_sentiment(): - emp_id, sixmonths = get_params() + emp_id = get_params() person = sentiment[sentiment.employee_id==emp_id].copy() - return pd.crosstab([person['timestamp'],person['workshop_name'],person['workshop_id']],person['score']).apply(lambda x: round(x/x.sum()*100,2), axis=1).reset_index().melt(id_vars=['timestamp','workshop_name','workshop_id']) + monyear_percent = pd.crosstab([person['timestamp'],person['workshop_name'],person['workshop_id']],person['sentiment']).apply(lambda x: round(x/x.sum()*100,2), axis=1).reset_index().melt(id_vars=['timestamp','workshop_name','workshop_id']) + return monyear_percent.sort_values(by='timestamp', ascending=False).reset_index(drop=True) def get_overall_reviews(monyear_percent): - emp_id, sixmonths = get_params() + emp_id = get_params() filter_idx = monyear_percent.groupby(['workshop_id'])['value'].transform(max) == monyear_percent['value'] al_reviews = monyear_percent[filter_idx].copy() - al_reviews.sort_values('score', ascending=False, inplace=True) + al_reviews.sort_values('sentiment', ascending=False, inplace=True) al_reviews.drop_duplicates(subset='workshop_id', keep="first", inplace=True) - person_reviews = reviews[(reviews.employee_id==emp_id) & (reviews.timestamp >= sixmonths)] + person_reviews = reviews[(reviews.employee_id==emp_id)] - sm_reviews = person_reviews[['satisfaction_score', 'workshop_id']].groupby(['workshop_id']).mean().round(1) + sm_reviews = person_reviews[['satisfaction_score', 'workshop_id', 'timestamp']].groupby(['workshop_id','timestamp']).mean().round(1) sm_reviews.reset_index(inplace=True) sm_reviews.loc[:,'workshop_name'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['workshop_name']) - sm_reviews.loc[:,'sentiment'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['score']) + sm_reviews.loc[:,'sentiment'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['sentiment']) sm_reviews.loc[:,'value'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['value']) return sm_reviews @@ -139,9 +112,8 @@ def vis_overall_sentiment(): chart = alt.Chart(monyear_percent).mark_bar().encode( y=alt.Y('value:Q', axis=alt.Axis(title='Percentage (%)')), x=alt.X('workshop_name:N', axis=alt.Axis(title='Workshop Name')), - color=alt.Color('score', scale=alt.Scale(domain=domain, range=colors), legend=alt.Legend(title="Sentiment")), - order="timestamp:T", - tooltip=[alt.Tooltip('value:Q', title="Percentage"), alt.Tooltip('score:N', title="Sentiment")] + color=alt.Color('sentiment', scale=alt.Scale(domain=domain, range=colors), legend=alt.Legend(title="Sentiment")), + tooltip=[alt.Tooltip('value:Q', title="Percentage"), alt.Tooltip('sentiment:N', title="Sentiment")] ).properties( width=800, height=300 ).configure_axis(grid=False) @@ -641,9 +613,7 @@ def gettimenow(): Response.workshop_id.in_(w.id for w in workshops), Response.comments != '').join( Workshop, isouter=True).order_by( Workshop.workshop_start.desc()).paginate( - per_page=30, page=1, error_out=True) - - # comments = pd.read_sql_query("SELECT workshop_id, comments FROM response", conn) + per_page=100, page=1, error_out=True) monyear_percent = get_overall_sentiment() diff --git a/app/forms.py b/app/forms.py index 3291bbe..2b8384b 100644 --- a/app/forms.py +++ b/app/forms.py @@ -3,6 +3,7 @@ from wtforms.fields.html5 import IntegerRangeField from wtforms.validators import ValidationError, DataRequired, Email, EqualTo from app.users import User +from joblib import load class LoginForm(FlaskForm): email = StringField('Work Email', validators=[DataRequired(), Email()]) diff --git a/app/models.py b/app/models.py index a2da9a5..f86ab72 100644 --- a/app/models.py +++ b/app/models.py @@ -69,3 +69,4 @@ class Response(db.Model): venue_score = db.Column(db.Integer) satisfaction_score = db.Column(db.Integer) comments = db.Column(db.Text) + sentiment = db.Column(db.String(10), nullable=True) diff --git a/app/routes.py b/app/routes.py index 985493b..5b42d30 100644 --- a/app/routes.py +++ b/app/routes.py @@ -6,6 +6,7 @@ from app.models import Employee, Workshop, Response from app.forms import LoginForm, RegistrationForm, SurveyForm, ResetPasswordRequestForm, ResetPasswordForm from app.email import send_pw_reset_email +from app.sentiment import preprocess_comments, predict_sentiment from datetime import datetime from sqlalchemy import func @@ -142,7 +143,8 @@ def rate(workshop_id): timeliness=form.time.data, venue_score=form.venue.data, satisfaction_score=form.satisfaction.data, - comments=form.comments.data + comments=preprocess_comments(form.comments.data), + sentiment=predict_sentiment(preprocess_comments(form.comments.data)) ) db.session.add(response) db.session.commit() diff --git a/app/sentiment.py b/app/sentiment.py new file mode 100644 index 0000000..ed8cea2 --- /dev/null +++ b/app/sentiment.py @@ -0,0 +1,18 @@ +from joblib import load +from pathlib import Path +import string, re + +def preprocess_comments(sentence): + sentence = sentence.lower() + sentence = sentence.translate(str.maketrans("","", string.punctuation)) + sentence = sentence.translate(str.maketrans("","", string.digits)) + sentence = re.sub(' +', ' ',sentence).strip() + if sentence == "": sentence = '-' + + return sentence + +def predict_sentiment(sentence): + my_model = load(str(Path().absolute())+'/model/sentiment/model.joblib') + word_vector = load(str(Path().absolute())+'/model/sentiment/vector.joblib') + + return str(my_model.predict(word_vector.transform([sentence]))[0]) \ No newline at end of file diff --git a/app/templates/accomplishment.html b/app/templates/accomplishment.html index b7a1161..8c4c832 100644 --- a/app/templates/accomplishment.html +++ b/app/templates/accomplishment.html @@ -157,7 +157,6 @@

rate_review Most Recent Reviews

{% endif %} {% endfor %} - diff --git a/app/templates/sub/single_response.html b/app/templates/sub/single_response.html index 296aab8..23af1e0 100644 --- a/app/templates/sub/single_response.html +++ b/app/templates/sub/single_response.html @@ -48,7 +48,7 @@
{{ review['sentiment'] }}
{% endif %} {% for comments in personstats['qualitative'].items %} - {% if (comments['workshop_id'] == review['workshop_id'] and comments['comments'] != '')%} + {% if (comments['workshop_id'] == review['workshop_id'] and comments['comments'] != '-')%}

{{comments['comments']}}