From 5c97a57d6af545da6361f18ac82025f17566cb15 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 21 Sep 2016 19:53:00 +0800 Subject: [PATCH] shell : url rewrite script , to retry generate url rewrite and clear discard rewrite url --- app/console/config/modules/Helper.php | 15 +++ app/console/modules/Helper/Module.php | 28 ++++++ .../controllers/UrlrewriteController.php | 99 +++++++++++++++++++ .../Product/controllers/PriceController.php | 2 +- models/mongodb/url/UrlRewrite.php | 4 +- services/Category.php | 5 +- services/Product.php | 2 +- services/Url.php | 13 ++- services/category/CategoryMongodb.php | 11 ++- services/product/ProductMongodb.php | 3 +- services/url/Rewrite.php | 4 + services/url/rewrite/RewriteMongodb.php | 25 ++++- services/url/rewrite/RewriteMysqldb.php | 10 +- shell/README.md | 11 ++- shell/urlRewrite.sh | 53 ++++++++++ 15 files changed, 268 insertions(+), 17 deletions(-) create mode 100644 app/console/config/modules/Helper.php create mode 100644 app/console/modules/Helper/Module.php create mode 100644 app/console/modules/Helper/controllers/UrlrewriteController.php create mode 100644 shell/urlRewrite.sh diff --git a/app/console/config/modules/Helper.php b/app/console/config/modules/Helper.php new file mode 100644 index 000000000..deac0ce3c --- /dev/null +++ b/app/console/config/modules/Helper.php @@ -0,0 +1,15 @@ + + [ + 'class' => '\fecshop\app\console\modules\Helper\Module', + ], +]; diff --git a/app/console/modules/Helper/Module.php b/app/console/modules/Helper/Module.php new file mode 100644 index 000000000..883c7a6c7 --- /dev/null +++ b/app/console/modules/Helper/Module.php @@ -0,0 +1,28 @@ + + * @since 1.0 + */ +class Module extends ConsoleModule +{ + public $blockNamespace; + public function init() + { + # 以下代码必须指定 + $nameSpace = __NAMESPACE__; + $this->controllerNamespace = $nameSpace . '\\controllers'; + $this->blockNamespace = $nameSpace . '\\block'; + parent::init(); + + } +} diff --git a/app/console/modules/Helper/controllers/UrlrewriteController.php b/app/console/modules/Helper/controllers/UrlrewriteController.php new file mode 100644 index 000000000..9385c8a17 --- /dev/null +++ b/app/console/modules/Helper/controllers/UrlrewriteController.php @@ -0,0 +1,99 @@ + + * @since 1.0 + */ +class UrlrewriteController extends Controller +{ + protected $_numPerPage = 50; + /** + * 寰楀埌褰撳墠鐨勬椂闂淬 + */ + public function actionNowtime(){ + echo time(); + } + + /** + * 寰楀埌浜у搧鐨勯〉鏁般 + */ + public function actionProductpagenum(){ + $count = Yii::$service->product->collCount($filter); + echo ceil($count/$this->_numPerPage); + } + + /** + * 寰楀埌浜у搧鐨勬绘暟銆 + */ + public function actionProductcount(){ + $count = Yii::$service->product->collCount($filter); + echo $count; + } + /** + * 澶勭悊浜у搧鐨勯噸鍐 + */ + public function actionProduct($pageNum){ + $filter['numPerPage'] = $this->_numPerPage; + $filter['pageNum'] = $pageNum; + $filter['asArray'] = true; + $products = Yii::$service->product->coll($filter); + $product_ids = []; + foreach($products['coll'] as $one){ + Yii::$service->product->save($one); + } + + } + + + + + /** + * 寰楀埌鍒嗙被鐨勯〉鏁般 + */ + public function actionCategorypagenum(){ + $count = Yii::$service->category->collCount($filter); + echo ceil($count/$this->_numPerPage); + } + + /** + * 寰楀埌鍒嗙被鐨勬绘暟銆 + */ + public function actionCategorycount(){ + $count = Yii::$service->category->collCount($filter); + echo $count; + } + /** + * 澶勭悊鍒嗙被鐨勯噸鍐 + */ + public function actionCategory($pageNum){ + $filter['numPerPage'] = $this->_numPerPage; + $filter['pageNum'] = $pageNum; + $filter['asArray'] = true; + $categorys = Yii::$service->category->coll($filter); + $category_ids = []; + foreach($categorys['coll'] as $one){ + Yii::$service->category->save($one); + } + + } + /** + * 鍒犻櫎鏃堕棿灏忎簬nowtime鐨 + */ + public function actionClearnoactive($nowtime){ + echo 'delete date gt '.$nowtime."\n"; + Yii::$service->url->rewrite->removeByUpdatedAt($nowtime); + } +} + + + diff --git a/app/console/modules/Product/controllers/PriceController.php b/app/console/modules/Product/controllers/PriceController.php index e94c51f3c..85954b471 100644 --- a/app/console/modules/Product/controllers/PriceController.php +++ b/app/console/modules/Product/controllers/PriceController.php @@ -32,7 +32,7 @@ class PriceController extends Controller public function actionComputefinalprice($pageNum){ $filter['numPerPage'] = $this->_numPerPage; $filter['pageNum'] = $pageNum; - $filter['asArray'] = false; + $filter['asArray'] = true; $products = Yii::$service->product->coll($filter); $product_ids = []; foreach($products['coll'] as $one){ diff --git a/models/mongodb/url/UrlRewrite.php b/models/mongodb/url/UrlRewrite.php index 65e298d0a..93a520fb5 100644 --- a/models/mongodb/url/UrlRewrite.php +++ b/models/mongodb/url/UrlRewrite.php @@ -31,7 +31,9 @@ public function attributes() 'type', 'custom_url_key', 'origin_url', - 'status' + 'status', + 'updated_at', + 'created_at', ]; } } \ No newline at end of file diff --git a/services/Category.php b/services/Category.php index d1c457e74..8c89d1722 100644 --- a/services/Category.php +++ b/services/Category.php @@ -53,6 +53,9 @@ protected function actionGetByPrimaryKey($primaryKey){ return $this->_category->getByPrimaryKey($primaryKey); } + protected function actionCollCount($filter=''){ + return $this->_category->collCount($filter); + } /** @@ -89,7 +92,7 @@ protected function actionGetTreeArr($rootCategoryId=0){ * @property $originUrlKey|String , 鍒嗙被鐨勫湪淇敼涔嬪墠鐨剈rl key.锛堝湪鏁版嵁搴撲腑淇濆瓨鐨剈rl_key瀛楁锛屽鏋滄病鏈夊垯涓虹┖锛 * 淇濆瓨鍒嗙被锛屽悓鏃剁敓鎴愬垎绫荤殑浼潤鎬乽rl锛堣嚜瀹氫箟url锛夛紝濡傛灉鎸夌収name鐢熸垚鐨剈rl鎴栬呰嚜瀹氫箟鐨剈rlkey瀛樺湪锛岀郴缁熷垯浼氬鍔犲嚑涓殢鏈烘暟瀛楀瓧绗︿覆锛屾潵澧炲姞鍞竴鎬с */ - protected function actionSave($one,$originUrlKey){ + protected function actionSave($one,$originUrlKey='catalog/category/index'){ return $this->_category->save($one,$originUrlKey); } /** diff --git a/services/Product.php b/services/Product.php index 36dd5b089..9fa706c6d 100644 --- a/services/Product.php +++ b/services/Product.php @@ -126,7 +126,7 @@ protected function actionGetCategoryProductIds($product_id_arr,$category_id){ * 淇濆瓨浜у搧锛堟彃鍏ュ拰鏇存柊锛夛紝浠ュ強淇濆瓨浜у搧鐨勮嚜瀹氫箟url * 濡傛灉鎻愪氦鐨勬暟鎹腑瀹氫箟浜嗚嚜瀹氫箟url锛屽垯鎸夌収鑷畾涔塽rl淇濆瓨鍒皍rlkey涓紝濡傛灉娌℃湁鑷畾涔塽rlkey锛屽垯浼氫娇鐢╪ame杩涜鐢熸垚銆 */ - protected function actionSave($one,$originUrlKey=''){ + protected function actionSave($one,$originUrlKey='catalog/product/index'){ return $this->_product->save($one,$originUrlKey); } /** diff --git a/services/Url.php b/services/Url.php index a9087918a..e375f5dcf 100644 --- a/services/Url.php +++ b/services/Url.php @@ -52,11 +52,16 @@ protected function actionSaveRewriteUrlKeyByStr($str,$originUrl,$originUrlKey,$t * if originUrlKey and originUrl is exist in url rewrite collectons. */ $model = $this->find(); - $data = $model->where([ + $data_one = $model->where([ 'custom_url_key' => $originUrlKey, 'origin_url' => $originUrl, - ])->asArray()->one(); - if(isset($data['custom_url_key'])){ + ])->one(); + if(isset($data_one['custom_url_key'])){ + /** + * 鍙杩涜浜嗘煡璇紝灏辫鏇存柊涓涓媟ewrite url琛ㄧ殑updated_at. + */ + $data_one->updated_at = time(); + $data_one->save(); return $originUrlKey; } } @@ -77,7 +82,9 @@ protected function actionSaveRewriteUrlKeyByStr($str,$originUrl,$originUrlKey,$t ]); if(!isset($UrlRewrite['origin_url'])){ $UrlRewrite = $this->newModel(); + $UrlRewrite->created_at = time(); } + $UrlRewrite->updated_at = time(); $UrlRewrite->type = $type; $UrlRewrite->custom_url_key = $urlKey; $UrlRewrite->origin_url = $originUrl; diff --git a/services/category/CategoryMongodb.php b/services/category/CategoryMongodb.php index 7674f523d..b611e8c50 100644 --- a/services/category/CategoryMongodb.php +++ b/services/category/CategoryMongodb.php @@ -57,12 +57,21 @@ public function coll($filter=''){ ]; } + /** + * 寰楀埌鎬绘暟 + */ + public function collCount($filter=''){ + $query = Category::find(); + $query = Yii::$service->helper->ar->getCollByFilter($query,$filter); + return $query->count(); + } + /** * @property $one|Array , save one data . 鍒嗙被鏁扮粍 * @property $originUrlKey|String , 鍒嗙被鐨勫湪淇敼涔嬪墠鐨剈rl key.锛堝湪鏁版嵁搴撲腑淇濆瓨鐨剈rl_key瀛楁锛屽鏋滄病鏈夊垯涓虹┖锛 * 淇濆瓨鍒嗙被锛屽悓鏃剁敓鎴愬垎绫荤殑浼潤鎬乽rl锛堣嚜瀹氫箟url锛夛紝濡傛灉鎸夌収name鐢熸垚鐨剈rl鎴栬呰嚜瀹氫箟鐨剈rlkey瀛樺湪锛岀郴缁熷垯浼氬鍔犲嚑涓殢鏈烘暟瀛楀瓧绗︿覆锛屾潵澧炲姞鍞竴鎬с */ - public function save($one,$originUrlKey){ + public function save($one,$originUrlKey='catalog/category/index'){ $currentDateTime = \fec\helpers\CDate::getCurrentDateTime(); $primaryVal = isset($one[$this->getPrimaryKey()]) ? $one[$this->getPrimaryKey()] : ''; if($primaryVal){ diff --git a/services/product/ProductMongodb.php b/services/product/ProductMongodb.php index bddb85b33..97dfb8f9e 100644 --- a/services/product/ProductMongodb.php +++ b/services/product/ProductMongodb.php @@ -98,7 +98,8 @@ public function getCategoryProductIds($product_id_arr,$category_id){ * 淇濆瓨浜у搧锛堟彃鍏ュ拰鏇存柊锛夛紝浠ュ強淇濆瓨浜у搧鐨勮嚜瀹氫箟url * 濡傛灉鎻愪氦鐨勬暟鎹腑瀹氫箟浜嗚嚜瀹氫箟url锛屽垯鎸夌収鑷畾涔塽rl淇濆瓨鍒皍rlkey涓紝濡傛灉娌℃湁鑷畾涔塽rlkey锛屽垯浼氫娇鐢╪ame杩涜鐢熸垚銆 */ - public function save($one,$originUrlKey=''){ + public function save($one,$originUrlKey='catalog/product/index'){ + if(!$this->initSave($one)){ return; } diff --git a/services/url/Rewrite.php b/services/url/Rewrite.php index 84c6da9bc..dc3f015bf 100644 --- a/services/url/Rewrite.php +++ b/services/url/Rewrite.php @@ -88,6 +88,10 @@ protected function actionRemove($ids){ return $this->_urlRewrite->remove($ids); } + protected function actionRemoveByUpdatedAt($time){ + return $this->_urlRewrite->removeByUpdatedAt($time); + } + protected function actionFind(){ return $this->_urlRewrite->find(); diff --git a/services/url/rewrite/RewriteMongodb.php b/services/url/rewrite/RewriteMongodb.php index 9ac57b988..4a844fae1 100644 --- a/services/url/rewrite/RewriteMongodb.php +++ b/services/url/rewrite/RewriteMongodb.php @@ -97,7 +97,6 @@ public function remove($ids){ $model = UrlRewrite::findOne($id); if(isset($model[$this->getPrimaryKey()]) && !empty($model[$this->getPrimaryKey()]) ){ $url_key = $model['url_key']; - Yii::$service->url->removeRewriteUrlKey($url_key); $model->delete(); }else{ //throw new InvalidValueException("ID:$id is not exist."); @@ -110,7 +109,6 @@ public function remove($ids){ $model = UrlRewrite::findOne($id); if(isset($model[$this->getPrimaryKey()]) && !empty($model[$this->getPrimaryKey()]) ){ $url_key = $model['url_key']; - Yii::$service->url->removeRewriteUrlKey($url_key); $model->delete(); }else{ Yii::$service->helper->errors->add("UrlRewrite Remove Errors:ID:$id is not exist."); @@ -121,6 +119,29 @@ public function remove($ids){ } + public function removeByUpdatedAt($time){ + if($time){ + UrlRewrite::deleteAll([ + '$or' => [ + [ + 'updated_at' => [ + '$lt' => (int)$time, + ], + ], + [ + 'updated_at' => [ + '$exists' => false + ] + ] + ] + + ]); + echo "delete complete \n"; + } + + } + + public function find(){ return UrlRewrite::find(); } diff --git a/services/url/rewrite/RewriteMysqldb.php b/services/url/rewrite/RewriteMysqldb.php index 489cda04c..8136f9585 100644 --- a/services/url/rewrite/RewriteMysqldb.php +++ b/services/url/rewrite/RewriteMysqldb.php @@ -124,7 +124,6 @@ public function remove($ids){ $model = UrlRewrite::findOne($id); if(isset($model[$this->getPrimaryKey()]) && !empty($model[$this->getPrimaryKey()]) ){ $url_key = $model['url_key']; - Yii::$service->url->removeRewriteUrlKey($url_key); $model->delete(); }else{ @@ -147,7 +146,6 @@ public function remove($ids){ $innerTransaction = Yii::$service->db->beginTransaction(); try { $url_key = $model['url_key']; - Yii::$service->url->removeRewriteUrlKey($url_key); $model->delete(); $innerTransaction->commit(); } catch (Exception $e) { @@ -163,6 +161,14 @@ public function remove($ids){ } + public function removeByUpdatedAt($time){ + if($time){ + UrlRewrite::deleteAll([ + '<','updated_at',$time + ]); + } + } + public function find(){ return UrlRewrite::find(); } diff --git a/shell/README.md b/shell/README.md index a6f986720..07eb54d52 100644 --- a/shell/README.md +++ b/shell/README.md @@ -26,13 +26,16 @@ Xun Search 鍙敮鎸佷腑鏂囷紝 ### 3.computeProductFinalPrice.sh 鏈剼鏈负璁$畻姣忎竴涓骇鍝佺殑final_price鐨勫硷紝 -鍚屾椂浼氬悓姝ヤ俊鎭埌鎼滅储琛紝浣嗘槸涓嶄細澶勭悊鍒犻櫎鎺夌殑琛 -鍥犳锛屾墽琛岃繖涓环鏍煎畬鎴愬悗锛岃繕闇瑕佹墽琛 fullSearchSync.sh鑴氭湰銆 - - +3.1 鍚屾椂浼氬悓姝ヤ俊鎭埌鎼滅储琛紝浣嗘槸涓嶄細鎵ц鍒犻櫎鎿嶄綔锛堣濡傛煇涓骇鍝 +鍒犻櫎浜嗭紝杩欎釜鑴氭湰鏄笉浼氭娴嬶紝鍚屾鍒犻櫎鐨勶紝濡傛灉瑕佸垹闄ら渶瑕佹墽琛 fullSearchSync.sh鑴氭湰銆傦級 +3.2 url rewrite鐨勪骇鍝佹暟鎹篃浼氬悓姝ワ紝鍚屾牱涓嶄細鎵ц鍒犻櫎url rewrite閲岄潰鐨勬暟鎹紙 +濡傛灉瑕佹竻闄ゆ畫鐣欑殑url rewrite鏁版嵁锛岃鎵цurlRewrite.sh鑴氭湰锛夈 +### 4.urlRewrite.sh鑴氭湰 +4.1 灏嗕骇鍝佺殑url鑷畾涔夐噸鏂拌窇涓娆°傚寘鎷垎绫伙紝浜у搧绛 +4.2 灏唘rlRewrite琛ㄤ腑娈嬬暀鐨勯噸鏂版暟鎹紝浣嗘槸鍦ㄤ骇鍝佸拰鍒嗙被涓笉瀛樺湪鐨勬竻绌恒 diff --git a/shell/urlRewrite.sh b/shell/urlRewrite.sh new file mode 100644 index 000000000..ec5faede9 --- /dev/null +++ b/shell/urlRewrite.sh @@ -0,0 +1,53 @@ +#!/bin/sh +Cur_Dir=$(cd `dirname $0`; pwd) +# init full search collection indexes. +$Cur_Dir/../../../../yii product/search/initindex +# get now update timestamp. +nowtime=`$Cur_Dir/../../../../yii helper/urlrewrite/nowtime` + +###### 1.Sync Section : Sync Product Serach Collection +# get product all count. +count=`$Cur_Dir/../../../../yii helper/urlrewrite/productcount` +pagenum=`$Cur_Dir/../../../../yii helper/urlrewrite/productpagenum` + + +echo "There are $count products to process" +echo "There are $pagenum products pages to process" +echo "##############ALL BEGINING###############"; +for (( i=1; i<=$pagenum; i++ )) +do + $Cur_Dir/../../../../yii helper/urlrewrite/product $i + echo "Page $i done" +done + + +#Category + +count=`$Cur_Dir/../../../../yii helper/urlrewrite/categorycount` +pagenum=`$Cur_Dir/../../../../yii helper/urlrewrite/categorypagenum` + + +echo "There are $count categorys to process" +echo "There are $pagenum categorys pages to process" +echo "##############ALL BEGINING###############"; +for (( i=1; i<=$pagenum; i++ )) +do + $Cur_Dir/../../../../yii helper/urlrewrite/category $i + echo "Page $i done" +done + + +#delete all search data that sync_updated_at $gt $nowtime. +$Cur_Dir/../../../../yii helper/urlrewrite/clearnoactive $nowtime + +###### 1.Sync Section End + + + + +echo "##############ALL COMPLETE###############"; + + + + +