From 55227cc493fd040ffeb59d604fc45c0a85d72baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=84=80=E1=85=B5=E1=86=B7=E1=84=8C=E1=85=A9=E1=86=BC?= =?UTF-8?q?=E1=84=8B=E1=85=AF=E1=86=AB=20Jongwon?= Date: Mon, 28 Aug 2023 01:51:02 +0900 Subject: [PATCH 1/2] =?UTF-8?q?[REFACTOR]=20#225:=20DataFrame=20=EC=BA=90?= =?UTF-8?q?=EC=8B=B1=EC=9C=BC=EB=A1=9C=20=EC=84=B1=EB=8A=A5=ED=96=A5?= =?UTF-8?q?=EC=83=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../__pycache__/recommend.cpython-311.pyc | Bin 4795 -> 5499 bytes .../redisConnection.cpython-311.pyc | Bin 0 -> 1895 bytes backend-recommend/recommend.py | 45 ++++++++++-------- backend-recommend/redisConnection.py | 22 +++++++++ 4 files changed, 48 insertions(+), 19 deletions(-) create mode 100644 backend-recommend/__pycache__/redisConnection.cpython-311.pyc create mode 100644 backend-recommend/redisConnection.py diff --git a/backend-recommend/__pycache__/recommend.cpython-311.pyc b/backend-recommend/__pycache__/recommend.cpython-311.pyc index 5f8599a3e2da9779b36f440c6377fd39f7073cdb..00ed3b43569047fb1427bd9aa34ec075e607f491 100644 GIT binary patch delta 2832 zcmai0`)}LE9Y0E>M2Vy*+Isu##1GkV);ybGO*1=jl(?>&Ho=zW$e zvwtC&1uB7>wKB9Wt5ih?z)&R;AQE%P;^F_Za%{3nzS@~{li91S$c?W$=S8kru2@Pn(1qXU{*8u!8Q8$2S1u~0)!{9>@r!4pIjVm{ls~O9k%4M@s zO0^t0#d2MgR4IGSa-~LOB`3!n7Kw)~C+<7Y%trWA@4%Q9u;&h@WGSgigNmdm>0DM0 zBLnPS8ALD+mYJ%_13!2C%roC=H104Pcl^Qed(UF4vG1t4@93iwKQK?6EuFh$oJgA| z(vPykW_Gl6T{E(}nbk|oZG*XOGPlPMmA$^ZofDmRx+c5E50)8zsyg5D-4YpihSmV7 z^_@T+H+t-SEg)$Y!0l8Ue8=&PyF;fn*D6H$@POXN=ToK72$C*-F<-fNXRN*rXZ-*PZW!Q3BSIQ5dCo6r`{tqx&>Y^F`MK5Zy0i zo!5AgzKpjC$i!q={pe@(3Ftx~tFhXIDFDza9k2}9MD-U2g+;(2Jtgmt=i1FZqLjf{*DK=Td-D^t+5 zW`E>(>9xFSS7i(!OOX?O3;Z0~>4)q0QmpGjVjZj!Eq6M{@^_anBJ449S zs{Re+V)8Tfz)PNj#YQ{KuiUNOwOTErJ-C7PI&=x@wK_z~5xqfc5JN9~>y2ovNoz!7 zTlHqGX)XW!hV&85@GoR7jV@D*R==ta1FdNz18DT9dJvwYVt-vo6f4%t;40yM>fBY#fzl8~JU#J-c{~3rA7I9s{3^vLkgzz8Uj^Nu!Q1F*z z94ziweD}-U<3NtU?M$5`-V9GO&4*Rlaw?K)@veqEeZhBEeZ^&+*C>gFS5D zYYy>(9KZ|UOJY&t`S)_87BOVGa#;z9ay?c%P{1yfjg#^X_$#)@k5`v_F&xIw1xapS z*kY8C^Y2N@FpdMwA65|w$`<%0*9?zwWW7zP#mZ8u=SKIioX*K<`68N*FSPOq-sWzO z#a949#q&g&=I*i+>?g+yv10fAIwRO-2HOm}-K5(~c74k7Q`DWl$-eR4vd=%Bcs%tZhG3sd&+Llhc}l4?M9%(40PP_DHoW4P_6`YF^5l5bQ01b`c$zOfOhkX#OgXZ!EeRujN`}64c zJb!3I_xbauLg*8vuxI+_%uPdRHihOD;PSNFknb)BV}%_vCkx4$)A_??K3Ev5n$YrE z*&iu~YG*U|GR0j+sLc$ub1eug??rZy7b6W=*FN+-3&1A<8)95y2Eu z_J?PE_k4v>!{285+e&s_aiXz*yMPd!@k}!_OrBci8Vs(<K8_3Qx68g-uU%yaz27Z7e#zAQyL+B1eqK5i@cGp64$eZdziRiY*4yZZarjr)T%) z$G@mCy7rr0`xk|!uAWj?&*H7eiIYo-lco1XBO#dy>2YFsDKT8Sdd*0Tnu$?l0{#B0 zV-(f00ubcMaxhwGn)zYgTMk9?OgYqu$naoVJlhbhZ1^!-zr@x*P>Op$y@jY_J59E; zWY;q7zspbX1;(Jan)KF^U7s>lQ~M02)?{i+WUYe#-TZOq_CtHXmwQ6Jj5A(kxrUQj zN=z#90r)5X(K9msn9HwWcpZa)z;cN>{BK9be;h38LMEwPwlATb1?*!lcVJ`u!LTpJ zxAYrVa^grvdQ-lFbT|nqWdI)tDQ~C*)++q3@co@Dq>~6D_A21*C5XBe@M^Rd_#Z3+ BzpMZN delta 2280 zcmZuxT}&L;6~42~@6OE5{;)s5E*RrrFm@gDqxuIsHZpbxTZB4F+-~d;U}h=H!otj8 zj5{N?k)vR=D%j(wH1WfzjT}26DpB%~sFFpcKJ;;hjKpdrq)3+4S1Ko})V}o2z#2ny zH22(d&pG$I=bU@){P)mbUupW3W&HrsXSGj-bqau|?xIZOw($Hsa<}RASu977F=++? z0ub;VI~GU>5a0tMBM{5vLj-`&;n*TiPiDFD8hZJ&!SYI=c6;x!`hVKmv$b8If7-ll zWp-Pa*@q|$42~^Ihs2N(*1yCeddkNI(h(tW8>A}*vPijem8oq7ES%G*ZP4)Pr3|IiG;*`_KfAfZ_p#)>o+NN(~iz_7>lU-;u4)oL|-w9)r{ zI~ZCG?8u~A1B=wIrPP!Vd&yR-Cb!y|076{w8#NwU6&-3HL3U>hd%QzA@zT-B-FEn{ z68g{RR@R~AslnGLl#%>6FMRbbU5}8%AN|p#$NKM?!$g}$-|2s2dV&dv_ZT!I4sug= z>#2X@>(%SoL($W4A-m7B09KE!{_^kb1@Ii`@3N_&!^i_UQQkU0Zcgd5?EY#|aVmzU zB!>`hibGS1=#Uvv8T!pYE~o#K?TASjM3QjG9CAbvB`H5GOG*7=pq3;@WEie2>iIxz z(m}KNk#WiW9X8iF=wM!$aWGjKkQ9m2{~YLP#Du{s`Dq8qI(U9ugrem3n&X2f;)>Wm zAk8>FA)}|bu3*wV(yfTltoT7Ye$Wav*`X$wCItZ@n>$xx zA`89qy-Pp0nArxKX)wKSvjQ4_!~q*B@^>y3FTtp&8jDKVRI1QbV!4I4=HFVpc&ESE zzs@$Uv5m&G#U8cUqlF6_6kSX%W{km4Mop^KqFQaLwa~SZ*k^=SDp${&3EoQZc7iw2 z=(E^Cv+2a@Ve_2!lMw@Qj1Qk^K7j2JzYqgxRTTXp5xz=*l zY&rXnnsq+C#;47#2P!JIj}s9bCvO;`j~8{fRkPCO06DdX4Q6L;3M> zaY%9SVbS$)P*>sL@cqbnNjeJsy=Ij5xsrRF*RKB|JIC_ER8Bl2WuVS|t#TZ4cs+>z zr^@R(9X+FcPs9q-w`Z65rLG6Ok+G6T?BtOZex>Vc{+o=|-fg#cn|;4F`>xoKrv@-=s`6BJHg;UE;^|MrO>0V$szP86+y*-q0m#wMa3H(b&b)au z@6FTu=6j!S0D;ttZ%9o*=r=y;it}V}6oC1if%nC4_3wI}&#i=eH>CA}&Cg;6)UXf9FdJ-*_j+CO#dzH90XhK0cwnxq^`P zGwh2Iyu$)@OVY&M-t}ont)wdwa2C%2|_P$74$WG zZ4wnx0af@JXM_mUOg1QBuJIc9;Bp3A8zQ`2*aysob?A5~KcqL@mW zDp@rmGY3_PW-!m9Nwt*uwKW;Abe#?2ifS@J)0t~-%T$fezhpw5Fb`D~lE`bR%$gbz znZ(^NZ<1Ri^yDgdWX|MzI;~QJXLNXY`0btOk^$3bM$3+_>baE-Jr|uay73D@ z2_3rqP500bGj;b=!#&k>PqmT5@!`QpEA-y}=9A4zpb@&+4BaeyTb@webG0=XYP-O| zva;Y0ZvQ8Tp3WNmC8LX%_J*E*(FjJG!ARM87zjRk_%z)J+-L@Fv^{5coO@K?$;ZdX zeVV?h!7usj-Pok~5`V;TQi@HxUe02`lwHhVV-sQ)wEvS3*tAO$%Gf4P6^^e#??_Pq z+St_ZYWcUnj4@~`-G}A0A?hcXWqHbX&$!RGSilwjm!A83ng zm#29Z&+y{w059Mf&V^ID-fG|Iq>9fsH7>$cp2Z?_LGe{kn!D_Jbp-=LPOT*J#4=}I zfj7aRK|T?5gZX>p{|cV6PSQCYD9_MybOea348E0TvpQK{Q$M34pyRYK@&IiSW86aS zlGR$sTe4aU4c5Cqd-Ye}wenKacdhd9VE(61>UWpwzU78*x#?Rj4YVaEhBv{63r-Ao KJqz?YIQ<*ad6)YD literal 0 HcmV?d00001 diff --git a/backend-recommend/recommend.py b/backend-recommend/recommend.py index 3bd8f7c..9456d78 100644 --- a/backend-recommend/recommend.py +++ b/backend-recommend/recommend.py @@ -4,6 +4,7 @@ from mlxtend.frequent_patterns import association_rules import pymysql import time +import redisConnection import os from dotenv import load_dotenv from flask import jsonify @@ -13,7 +14,7 @@ def mysql_create_session(): conn = pymysql.connect(host=os.getenv('host'), user=os.getenv('user'), password=os.getenv('password'), db=os.getenv('db')) cur = conn.cursor() - return conn, cur + return conn, cur def recByApriori(body): start = time.time() @@ -32,26 +33,32 @@ def recByApriori(body): input = set(input) dataset = [] - conn, cur = mysql_create_session() - try: - cur.execute('SELECT hm.history_id, sh.sold_count, sh.sold_options_id FROM SalesHistory sh INNER JOIN HistoryModelMapper hm ON sh.history_id = hm.history_id WHERE sh.car_id = %s AND hm.model_id IN (%s, %s, %s) GROUP BY hm.history_id HAVING COUNT(DISTINCT hm.model_id) = 3;', (carId, powerTrainId, bodyTypeId, operationId)) - dbRow = cur.fetchall() - finally: - conn.close() - - for j in range(len(dbRow)): - oneRow = dbRow[j][2] - if(oneRow == ''): - continue - options = oneRow.split(",") - for i in range(int(dbRow[j][1])): - dataset.append(options) + df = redisConnection.redis_getData() + if df is None: + conn, cur = mysql_create_session() + try: + cur.execute('SELECT hm.history_id, sh.sold_count, sh.sold_options_id FROM SalesHistory sh INNER JOIN HistoryModelMapper hm ON sh.history_id = hm.history_id WHERE sh.car_id = %s AND hm.model_id IN (%s, %s, %s) GROUP BY hm.history_id HAVING COUNT(DISTINCT hm.model_id) = 3;', (carId, powerTrainId, bodyTypeId, operationId)) + dbRow = cur.fetchall() + finally: + conn.close() + + for j in range(len(dbRow)): + oneRow = dbRow[j][2] + if(oneRow == ''): + continue + options = oneRow.split(",") + for i in range(int(dbRow[j][1])): + dataset.append(options) + + start = time.time() + te = TransactionEncoder() + te_ary = te.fit(dataset).transform(dataset) + df = pd.DataFrame(te_ary, columns=te.columns_) + redisConnection.redis_setData(df) - start = time.time() - te = TransactionEncoder() - te_ary = te.fit(dataset).transform(dataset) - df = pd.DataFrame(te_ary, columns=te.columns_) df = df.iloc[:100000] + + frequent_itemsets = apriori(df, min_support=0.01, use_colnames=True) result_itemsets = association_rules(frequent_itemsets, metric="confidence", min_threshold=0.1) diff --git a/backend-recommend/redisConnection.py b/backend-recommend/redisConnection.py new file mode 100644 index 0000000..ec71b68 --- /dev/null +++ b/backend-recommend/redisConnection.py @@ -0,0 +1,22 @@ +import io +import redis +import pandas as pd +import os + +redis_pool = redis.ConnectionPool(host='3.37.253.113', port=6379, password=1234, db=0) + +def redis_getData(): + with redis.StrictRedis(connection_pool=redis_pool) as conn: + bytes_object = conn.get("rec_dataframe") + if bytes_object is None: + return None + buffer = io.BytesIO(bytes_object) + return pd.read_pickle(buffer) + +def redis_setData(df): + with redis.StrictRedis(connection_pool=redis_pool) as conn: + buffer = io.BytesIO() + df.to_pickle(buffer) + buffer.seek(0) + bytes_object = buffer.read() + conn.set('rec_dataframe', bytes_object, 60*60*24) From fcca59e0ba9147ab03abadc3f3812e90b96f46a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=84=80=E1=85=B5=E1=86=B7=E1=84=8C=E1=85=A9=E1=86=BC?= =?UTF-8?q?=E1=84=8B=E1=85=AF=E1=86=AB=20Jongwon?= Date: Mon, 28 Aug 2023 01:57:42 +0900 Subject: [PATCH 2/2] =?UTF-8?q?[FEAT]=20#225:=20Cache=20TTL=20=EC=84=A4?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../__pycache__/recommend.cpython-311.pyc | Bin 5499 -> 5499 bytes .../redisConnection.cpython-311.pyc | Bin 1895 -> 1895 bytes backend-recommend/redisConnection.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) diff --git a/backend-recommend/__pycache__/recommend.cpython-311.pyc b/backend-recommend/__pycache__/recommend.cpython-311.pyc index 00ed3b43569047fb1427bd9aa34ec075e607f491..18dff80df2618a0ca6c1a043b4982a31e7a75915 100644 GIT binary patch delta 19 ZcmeyZ^;?T;IWI340}#Be-^f)c3IISr1G@XqRohVWUIt~B}^)594 delta 170 zcmaFP_neP+IWI340}x!OeVwv;Bd;DSQy}AHTh@L?tI20s-P1WPa#&vBu)M%w`GK32 q)#?KSjQ9v*eSvck64+Qje*C~Dj1Ze#&2~u&XgV7sI#HwybQ}Of=Pq;r diff --git a/backend-recommend/redisConnection.py b/backend-recommend/redisConnection.py index ec71b68..3546f36 100644 --- a/backend-recommend/redisConnection.py +++ b/backend-recommend/redisConnection.py @@ -19,4 +19,4 @@ def redis_setData(df): df.to_pickle(buffer) buffer.seek(0) bytes_object = buffer.read() - conn.set('rec_dataframe', bytes_object, 60*60*24) + conn.set('rec_dataframe', bytes_object, 60*60*24*7)