diff --git "a/2017/05/07/\347\216\257\345\242\203\345\256\211\350\243\205/\345\234\250Hadoop\345\237\272\347\241\200\344\270\212Hive\347\232\204\345\256\211\350\243\205/index.html" "b/2017/05/07/\347\216\257\345\242\203\345\256\211\350\243\205/\345\234\250Hadoop\345\237\272\347\241\200\344\270\212Hive\347\232\204\345\256\211\350\243\205/index.html"
index 3417bca1..96eacf11 100644
--- "a/2017/05/07/\347\216\257\345\242\203\345\256\211\350\243\205/\345\234\250Hadoop\345\237\272\347\241\200\344\270\212Hive\347\232\204\345\256\211\350\243\205/index.html"
+++ "b/2017/05/07/\347\216\257\345\242\203\345\256\211\350\243\205/\345\234\250Hadoop\345\237\272\347\241\200\344\270\212Hive\347\232\204\345\256\211\350\243\205/index.html"
@@ -31,8 +31,8 @@
 <meta property="article:modified_time" content="2024-06-15T08:01:01.810Z">
 <meta property="article:author" content="wnma3mz">
 <meta property="article:tag" content="Hadoop">
-<meta property="article:tag" content="大数据">
 <meta property="article:tag" content="Hive">
+<meta property="article:tag" content="大数据">
 <meta name="twitter:card" content="summary">
 
 <link rel="canonical" href="https://wnma3mz.github.io/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/">
@@ -382,8 +382,8 @@ <h3 id="转换类型">转换类型</h3>
       <footer class="post-footer">
           <div class="post-tags">
               <a href="/hexo_blog/tags/Hadoop/" rel="tag"># Hadoop</a>
-              <a href="/hexo_blog/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/" rel="tag"># 大数据</a>
               <a href="/hexo_blog/tags/Hive/" rel="tag"># Hive</a>
+              <a href="/hexo_blog/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/" rel="tag"># 大数据</a>
           </div>
 
         
diff --git "a/2018/04/14/\351\230\205\350\257\273\347\254\224\350\256\260/\347\245\236\347\273\217\347\275\221\347\273\234\347\254\224\350\256\260\342\205\240/index.html" "b/2018/04/14/\351\230\205\350\257\273\347\254\224\350\256\260/\347\245\236\347\273\217\347\275\221\347\273\234\347\254\224\350\256\260\342\205\240/index.html"
index 85a99f85..43b41817 100644
--- "a/2018/04/14/\351\230\205\350\257\273\347\254\224\350\256\260/\347\245\236\347\273\217\347\275\221\347\273\234\347\254\224\350\256\260\342\205\240/index.html"
+++ "b/2018/04/14/\351\230\205\350\257\273\347\254\224\350\256\260/\347\245\236\347\273\217\347\275\221\347\273\234\347\254\224\350\256\260\342\205\240/index.html"
@@ -31,9 +31,9 @@
 <meta property="article:modified_time" content="2024-06-15T08:01:01.818Z">
 <meta property="article:author" content="wnma3mz">
 <meta property="article:tag" content="Python">
-<meta property="article:tag" content="机器学习">
 <meta property="article:tag" content="神经网络">
 <meta property="article:tag" content="NumPy">
+<meta property="article:tag" content="机器学习">
 <meta name="twitter:card" content="summary">
 
 <link rel="canonical" href="https://wnma3mz.github.io/2018/04/14/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A0/">
@@ -401,9 +401,9 @@ <h2 id="改进神经网络v3.0dropout">改进神经网络v3.0——dropout</h2>
       <footer class="post-footer">
           <div class="post-tags">
               <a href="/hexo_blog/tags/Python/" rel="tag"># Python</a>
-              <a href="/hexo_blog/tags/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/" rel="tag"># 机器学习</a>
               <a href="/hexo_blog/tags/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C/" rel="tag"># 神经网络</a>
               <a href="/hexo_blog/tags/NumPy/" rel="tag"># NumPy</a>
+              <a href="/hexo_blog/tags/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/" rel="tag"># 机器学习</a>
           </div>
 
         
diff --git a/2022/12/31/ShortPaperReading/index.html b/2022/12/31/ShortPaperReading/index.html
index ad61353c..a27a9590 100644
--- a/2022/12/31/ShortPaperReading/index.html
+++ b/2022/12/31/ShortPaperReading/index.html
@@ -28,7 +28,7 @@
 <meta property="og:description" content="读研期间，短时 paper reading">
 <meta property="og:locale">
 <meta property="article:published_time" content="2022-12-31T13:12:43.000Z">
-<meta property="article:modified_time" content="2024-09-14T13:40:27.932Z">
+<meta property="article:modified_time" content="2024-09-14T14:11:05.737Z">
 <meta property="article:author" content="wnma3mz">
 <meta property="article:tag" content="Federated Learning">
 <meta name="twitter:card" content="summary">
@@ -229,7 +229,7 @@ <h1 class="post-title" itemprop="name headline">
                   <i class="far fa-calendar-check"></i>
                 </span>
                 <span class="post-meta-item-text">更新于</span>
-                <time title="عُدل：2024-09-14 21:40:27" itemprop="dateModified" datetime="2024-09-14T21:40:27+08:00">2024-09-14</time>
+                <time title="عُدل：2024-09-14 22:11:05" itemprop="dateModified" datetime="2024-09-14T22:11:05+08:00">2024-09-14</time>
               </span>
             <span class="post-meta-item">
               <span class="post-meta-item-icon">
@@ -274,16 +274,21 @@ <h2 id="inspirer">Inspirer</h2>
 <p>实验结果证明，该方法能够在精度不变的前提下，减少50倍的通信规模，并且提升了22%的速度</p>
 <p>这篇文章可以间接为受控共享学习中，之前的实验结果做一个解释，只接受梯度值较大的客户端，即表明该客户端的贡献较大。</p>
 <p>问题：分布式机器学习的任务，数据可能都是来源同一个域内的，甚至就是相同的数据；从该文中的图来看，不能证明之前实验收敛速度加快的原因</p>
+<blockquote>
 <p>Alham Fikri Aji and Kenneth Heafield. Sparse communication for
 distributed gradient descent. In Empirical Methods in Natural Language
 Processing (EMNLP), 2017.</p>
+</blockquote>
 <h1 id="knowledge-distillation-with-svd">Knowledge Distillation with
 SVD</h1>
 <p>2020.10.23</p>
-<p>Knowledge Distillation (KD) - Fully Connected Layers-&gt;L2 loss</p>
-<p>KD-FSP - Feature Map-&gt;L2 loss</p>
-<p>KD-SVD - Feature Map-&gt;SVD</p>
-<p>KD-EID - KD-SVD(Adaptively)</p>
+<ul>
+<li><p>Knowledge Distillation (KD): Fully Connected Layers-&gt;L2
+loss</p></li>
+<li><p>KD-FSP: Feature Map-&gt;L2 loss</p></li>
+<li><p>KD-SVD: Feature Map-&gt;SVD</p></li>
+<li><p>KD-EID: KD-SVD(Adaptively)</p></li>
+</ul>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image3.png" /></p>
 <p>基于SVD对模型进行压缩的新改进。</p>
@@ -292,9 +297,11 @@ <h1 id="knowledge-distillation-with-svd">Knowledge Distillation with
 Map进行SVD，这样可以有效提取教师模型中的知识，更准确地传递给学生信息。</p>
 <p>新发表的文章，则是对这种方法进行了改进，无需人工干预参数设计。实验证明该方法也是有效的（在TinyImageNet对比原始的学生模型提高了2.89%）。文章大部分内容都还是ECCV那篇文章的内容，增加的自适应部分是判断教师与学生模型的Feature
 Map大小差异，来决定使用SVD还是特征分解。</p>
+<blockquote>
 <p>Lee S, Song B C. Knowledge Transfer via Decomposing Essential
 Information in Convolutional Neural Networks[J]. IEEE transactions on
 neural networks and learning systems.</p>
+</blockquote>
 <h1 id="knowledge-distillation-in-federated-learning">Knowledge
 Distillation in Federated Learning</h1>
 <p>2020.10.30</p>
@@ -316,12 +323,14 @@ <h2 id="understanding">Understanding</h2>
 <p>主要贡献是提出了鲁棒性的联邦蒸馏框架，并且在CV和NLP任务上都得到了验证，还对异质模型，无标签数据等进行了大量实验。</p>
 <p>右图是文章提出的算法（图是自己画的，原文并没有给方法图）,首先是根据不同的客户端在本地训练得到梯度值后，上传至服务端进行聚合，后发回给客户端进行作为教师模型进行KL散度计算，对比于之前的FedMD的方法是不加入公开数据集部分。FedDF这个方法跟之前三室的一位师姐介绍在物联网中的应用，框架基本一致。</p>
 <p>FedDF is designed for effective model fusion on the server,
-considering the accuracy of the global model on the test dataset.
-FedDF是为了有效融合模型，考虑了全局模型在测试集上的准确性，所以未对一些实验进行比较（比如FedMD）。我觉得在找借口，FedMD是有这部分实验的。</p>
+considering the accuracy of the global model on the test dataset.</p>
+<p>FedDF是为了有效融合模型，考虑了全局模型在测试集上的准确性，所以未对一些实验进行比较（比如FedMD）。我觉得在找借口，FedMD是有这部分实验的。</p>
 <p>文章为了辅助理解这个框架，还加入了泛化边界，数据集的分布与大小，蒸馏的实验（类似于消融实验）</p>
+<blockquote>
 <p>Lin T, Kong L, Stich S U, et al. Ensemble Distillation for Robust
 Model Fusion in Federated Learning[J]. arXiv preprint arXiv:2006.07242,
 2020.</p>
+</blockquote>
 <p>洛桑联邦理工学院</p>
 <h2 id="feddf-experiments">FedDF Experiments</h2>
 <p>2020.12.04</p>
@@ -347,31 +356,23 @@ <h2 id="feddf-experiments">FedDF Experiments</h2>
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image5.png" /></p>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image6.png" /></p>
+<blockquote>
 <p>Lin T, Kong L, Stich S U, et al. Ensemble Distillation for Robust
 Model Fusion in Federated Learning[J]. arXiv preprint arXiv:2006.07242,
 2020.</p>
+</blockquote>
 <h1
 id="ensemble-distillation-for-robust-model-fusion-in-federated-learning">Ensemble
 Distillation for Robust Model Fusion in Federated Learning</h1>
 <p>2021.01.15</p>
 <h2 id="联邦学习知识蒸馏">联邦学习——知识蒸馏</h2>
 <ul>
-<li>Zero-Shot（无标签）
-<ul>
-<li>利用公开数据集，训练GAN生成无标签的数据</li>
-</ul></li>
-<li>常规迁移学习
-<ul>
-<li>利用客户端模型分类层权重生成数据（Dirichlet分布）</li>
-</ul></li>
-<li>噪声迁移学习
-<ul>
-<li>差分隐私+联邦学习</li>
-</ul></li>
-<li>SWA迁移学习
-<ul>
-<li>在一般的优化器上增加SWA，可以提高性能</li>
-</ul></li>
+<li><p>Zero-Shot（无标签）:
+利用公开数据集，训练GAN生成无标签的数据</p></li>
+<li><p>常规迁移学习:
+利用客户端模型分类层权重生成数据（Dirichlet分布）</p></li>
+<li><p>噪声迁移学习: 差分隐私+联邦学习</p></li>
+<li><p>SWA迁移学习: 在一般的优化器上增加SWA，可以提高性能</p></li>
 </ul>
 <figure>
 <img
@@ -390,9 +391,11 @@ <h2 id="联邦学习知识蒸馏">联邦学习——知识蒸馏</h2>
 target="_blank" rel="noopener" href="https://www.zhihu.com/question/26751755">Dirichlet分布</a>；<a
 target="_blank" rel="noopener" href="https://arxiv.org/abs/2009.05537">差分隐私+联邦学习</a>；<a
 target="_blank" rel="noopener" href="https://pytorch.org/blog/stochastic-weight-averaging-in-pytorch/">SWA</a></p>
+<blockquote>
 <p>Lin T, Kong L, Stich S U, et al. Ensemble distillation for robust
 model fusion in federated learning[J]. arXiv preprint arXiv:2006.07242,
 2020.</p>
+</blockquote>
 <h1
 id="group-knowledge-transferfederated-learning-of-large-cnns-at-the-edge">Group
 Knowledge Transfer:Federated Learning of Large CNNs at the Edge</h1>
@@ -420,8 +423,8 @@ <h2 id="methodgroup-knowledge-transfer">Method(Group Knowledge
 <li><p>FedPAV满足了需求，但是对于性能上不能完成满足（小数据提升，大数据集下降，训练过程振荡）</p></li>
 <li><p>FedPAV+KD满足需求，提升了性能，但是不稳定（性能不稳定，训练过程稳定）</p>
 <ol type="1">
-<li>Tricks：正则化</li>
-<li>Tricks：Cosing Distance Weight</li>
+<li>Trick：正则化</li>
+<li>Trick：Cosing Distance Weight</li>
 </ol></li>
 </ol>
 <p><img
@@ -463,7 +466,9 @@ <h2 id="work">Work</h2>
 nonconvex settings.</p>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image18.png" /></p>
+<blockquote>
 <p>Google: 2020 Feb-2020 Dec</p>
+</blockquote>
 <h1
 id="preservation-of-the-global-knowledge-by-not-true-self-knowledge-distillation-in-federated-learning">Preservation
 of the Global Knowledge by Not-True Self Knowledge Distillation in
@@ -484,9 +489,11 @@ <h2 id="研究">研究</h2>
 <p>有偏局部特征转移诱导拟合</p>
 <p>遗忘灾难：把CIFAR10输入拉成二维，在进行100轮联邦学习后，估计数据的概率密度函数，上图这个分布存在明显偏差；提出的方法基本无偏差</p>
 <p>研究方法：将传统的训练，改成了蒸馏的方式（由于是模型架构相同，所以可以叫做自蒸馏），也在本地进行蒸馏；另外，基于IJCAJ2021知识蒸馏的工作，对教师模型预测正确的标签剔除，再KL散度计算，以进行改进。</p>
+<blockquote>
 <p>Taehyeon Kim, Jaehoon Oh, NakYil Kim, Sangwook Cho, and Se-Young Yun.
 Comparingkullback-leibler divergence and mean squared error loss in
 knowledge distillation.arXiv preprintarXiv:2105.08919, 2021.</p>
+</blockquote>
 <h1
 id="preservation-of-the-global-knowledge-by-not-trueself-knowledge-distillation-in-federated-learning">Preservation
 of the Global Knowledge by Not-TrueSelf Knowledge Distillation in
@@ -505,8 +512,10 @@ <h2 id="研究">研究</h2>
 <p>全局模型的预测可以作为先前数据分布的参考，从而产生类似于在CL中使用情景记忆的效果。</p>
 <p>beta是指KL散度损失的权重</p>
 <p>结果表明，数据分布的正确性，即数据分布的偏态性，可能影响数据的准确性。</p>
+<blockquote>
 <p>Zhao Y, Li M, Lai L, et al. Federated learning with non-iid data[J].
 arXiv preprint arXiv:1806.00582, 2018.</p>
+</blockquote>
 <h1 id="learn-distributed-gan-with-temporary-discriminators">Learn
 distributed GAN with Temporary Discriminators</h1>
 <p>2021.07.02</p>
@@ -539,10 +548,12 @@ <h2 id="hessian-free">Hessian-Free</h2>
 的hat，是提出的问题建模。从w
 hat开始训练，更容易适用于大部分任务。即原始公式</p>
 <p>根据原始公式，得到两种近似，一阶形式，与海森形式（二阶）。</p>
+<blockquote>
 <p>Alireza Fallah, Aryan Mokhtari, Asuman Ozdaglar. Personalized
 Federated Learning with Theoretical Guarantees: A Model-Agnostic
 Meta-Learning Approach. In Advances in Neural Information Processing
 Systems (NIPS), 2020(33): 3557–3568.</p>
+</blockquote>
 <h1 id="distilled-one-shot-federated-learning">Distilled One-Shot
 Federated Learning</h1>
 <p>2021.08.20</p>
@@ -576,9 +587,13 @@ <h2 id="方法">方法</h2>
 <p>randomly adjusts: 在每次学习过程中，进行随机的调整</p>
 <p>Random masking randomly selects a fraction p_{rm} of the distilled
 data at each training iteration andreplaces it with a random tensor.</p>
+<blockquote>
+<p>NSF Center for Big Learning University of Florida</p>
+</blockquote>
+<blockquote>
 <p>Zhou Y, Pu G, Ma X, et al. Distilled one-shot federated learning[J].
 arXiv preprint arXiv:2009.07999, 2020.</p>
-<p>NSF Center for Big Learning University of Florida</p>
+</blockquote>
 <h1 id="does-knowledge-distillation-really-work">Does Knowledge
 Distillation Really Work?</h1>
 <p>2021.08.27</p>
@@ -595,17 +610,23 @@ <h2 id="generalization-vs-fidelity">Generalization vs Fidelity</h2>
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image33.png" /></p>
 <h2 id="why-low-fidelity">Why Low Fidelity</h2>
 <ul>
-<li>Architecture: ResNet √ ; VGG ×</li>
-<li>Student Capacity: ×</li>
-<li>Identifiability：distillation dataset ≠ test dataset</li>
-<li>Optimization：√ <img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image32.png" /></li>
+<li>Architecture: ResNet <span class="math inline">\(\sqrt{}\)</span> ;
+VGG <span class="math inline">\(\times\)</span></li>
+<li>Student Capacity: <span class="math inline">\(\times\)</span></li>
+<li>Identifiability：distillation dataset <span
+class="math inline">\(\neq\)</span> test dataset</li>
+<li>Optimization：<span class="math inline">\(\sqrt{}\)</span></li>
 </ul>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image32.png" /></p>
 <p>知识蒸馏在深度学习中被广泛应用，但其是否真的有效？简单来说，KD确实有效，能够提升学生网络模型的性能；然而教师往往只能传递有限的知识给学生。
 文章提出了一个新的名词Agreement，或者说Fidelity，即教师与学生对于数据的预测相似度。</p>
-<p>为什么要提出一个新的概念，根据图1中的实验结果xxx</p>
-<p>为什么会有更低的保真度 1. 优化器。随着epoch的增加，一致性增加 2.
-初始化。教师权重+随机权重，lambda比值，越大表示趋近于教师</p>
+<p>为什么要提出一个新的概念，根据图1中的实验结果</p>
+<p>为什么会有更低的保真度</p>
+<ol type="1">
+<li><p>优化器。随着epoch的增加，一致性增加</p></li>
+<li><p>初始化。教师权重+随机权重，lambda比值，越大表示趋近于教师</p></li>
+</ol>
 <p>我们发现，如果学生的初始化距离教师很远（λ≤0.25），优化器会收敛到蒸馏损失的次优值，从而产生与教师明显不同的学生。然而在λ=
 0.375
 处有一个突然的变化。最终的训练损失下降到最优值并且一致性急剧增加，并且行为继续
@@ -615,9 +636,13 @@ <h2 id="why-low-fidelity">Why Low Fidelity</h2>
 0.375）时，学生收敛到与老师相同的盆地，达到接近 100% 的一致性</p>
 <p>我们终于确定了之前所有干预措施无效的根本原因。知识蒸馏无法收敛到最佳学生参数，即使我们知道一个解决方案并在优化方向上给初始化一个小的开端。事实上，虽然可识别性可能是一个问题，但为了在所有输入上匹配教师，学生有
 至少在用于蒸馏的数据上匹配老师，并实现蒸馏损失的接近最优值。在实践中，优化收敛到次优解，导致蒸馏保真度不佳</p>
+<blockquote>
+<p>Google Research</p>
+</blockquote>
+<blockquote>
 <p>Stanton S, Izmailov P, Kirichenko P, et al. Does Knowledge
 Distillation Really Work?[J]. arXiv preprint arXiv:2106.05945, 2021.</p>
-<p>Google Research</p>
+</blockquote>
 <h1 id="model-contrastive-federated-learning">Model-Contrastive
 Federated Learning</h1>
 <p>2021.09.18</p>
@@ -633,10 +658,14 @@ <h2 id="method-1">Method</h2>
 b})/\tau)+\exp(\mathrm{sim}(z,z_{p r e e v})/\tau)}\]</span></p>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image36.png" /></p>
+<blockquote>
+<p>UC Berkeley</p>
+</blockquote>
+<blockquote>
 <p>Li Q, He B, Song D. Model-Contrastive Federated
 Learning[C]//Proceedings of the IEEE/CVF Conference on Computer Vision
 and Pattern Recognition. 2021: 10713-10722.</p>
-<p>UC Berkeley</p>
+</blockquote>
 <h1
 id="data-free-knowledge-distillation-for-heterogeneous-federated-learning">Data-Free
 Knowledge Distillation for Heterogeneous Federated Learning</h1>
@@ -652,10 +681,14 @@ <h2 id="method-2">Method</h2>
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image38.png" /></p>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image40.png" /></p>
+<blockquote>
+<p>Michigan State University</p>
+</blockquote>
+<blockquote>
 <p>Zhuangdi Zhu and Junyuan Hong and Jiayu Zhou. 2021. Data-Free
 Knowledge Distillation for Heterogeneous Federated Learning. In ICML,
 139:12878-12889.</p>
-<p>Michigan State University</p>
+</blockquote>
 <h1 id="beyond-sharing-weights-for-deep-domain-adaptation">Beyond
 Sharing Weights for Deep Domain Adaptation</h1>
 <p>2021.10.18</p>
@@ -669,9 +702,11 @@ <h2 id="method-3">Method</h2>
 <h2 id="discussion">Discussion</h2>
 <p>It therefore seems reasonable that thehigher layers of the network,
 which encode higher-level in-formation, should be domain-specific.</p>
+<blockquote>
 <p>Rozantsev, A., Salzmann, M., &amp; Fua, P. (2019). Beyond Sharing
 Weights for Deep Domain Adaptation. IEEE Transactions on Pattern
 Analysis and Machine Intelligence, 41(4), 801-814.</p>
+</blockquote>
 <h1
 id="feddg-federated-domain-generalization-on-medical-image-segmentation-via-episodic-learning-in-continuous-frequency-space">FedDG:
 Federated Domain Generalization on Medical Image Segmentation via
@@ -711,11 +746,15 @@ <h2 id="method-4">Method</h2>
 <p>由于训练背景是医学图像处理，涉及到语义分割，所以需要使用常规的分割损失函数。再考虑图形的边界与背景，对应右边的绿色部分，采用InfoNCE损失函数
 对边界与背景做提取处理，计算InfoNCE损失。hm，hp表示同一类的正面特征，表示边界与背景。</p>
 <p>所以整个训练过程就是利用傅里叶变换提取分布信息，结合一般的分割与InfoNCE损失进行训练。主要的创新点在傅里叶变换提取了可共享的信息。下一次再进行具体的方法介绍。</p>
+<blockquote>
+<p>CUHK, 香港中文大学</p>
+</blockquote>
+<blockquote>
 <p>Liu Q, Chen C, Qin J, et al. Feddg: Federated domain generalization
 on medical image segmentation via episodic learning in continuous
 frequency space. Proceedings of the IEEE/CVF Conference on Computer
 Vision and Pattern Recognition. 2021: 1013-1023.</p>
-<p>CUHK, 香港中文大学</p>
+</blockquote>
 <h1
 id="refine-myself-by-teaching-myself-feature-refinement-via-self-knowledge-distillation">Refine
 Myself by Teaching Myself : Feature Refinement via Self-Knowledge
@@ -741,10 +780,12 @@ <h2 id="result-1">Result</h2>
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image57.png" /></p>
 <p><a
 target="_blank" rel="noopener" href="https://github.com/MingiJi/FRSKD">https://github.com/MingiJi/FRSKD</a></p>
+<blockquote>
 <p>Ji M, Shin S, Hwang S, et al. Refine Myself by Teaching Myself:
 Feature Refinement via Self-Knowledge Distillation. Proceedings of the
 IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2021:
 10664-10673.</p>
+</blockquote>
 <h1
 id="no-fear-of-heterogeneity-classifier-calibration-for-federated-learning-with-non-iid-data">No
 Fear of Heterogeneity: Classifier Calibration for Federated Learning
@@ -761,9 +802,11 @@ <h2 id="method-6">Method</h2>
 <h2 id="result-2">Result</h2>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image62.png" /></p>
+<blockquote>
 <p>Luo M, Chen F, Hu D, et al. No Fear of Heterogeneity: Classifier
 Calibration for Federated Learning with Non-IID Data[J]. arXiv preprint
 arXiv:2106.05001, 2021.</p>
+</blockquote>
 <h1 id="federated-learning-with-personalization-layers">Federated
 Learning with Personalization Layers</h1>
 <p>2021.12.02</p>
@@ -778,9 +821,11 @@ <h2 id="method-7">Method</h2>
 R}\big(\displaystyle\mathrm{W}_B,\displaystyle\mathrm{W}_{P}\big)\triangleq\frac{1}{n_{j}}\sum_{i=1}^{n_{j}}l(y_{j,i},f\big(x_{j,i};\displaystyle\mathrm{W}_B,\displaystyle\mathrm{W}_{P}\big)\big)\]</span></p>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image67.png" /></p>
+<blockquote>
 <p>Manoj Ghuhan Arivazhagan, Vinay Aggarwal, Aaditya Kumar Singh, and
 Sunav Choudhary. 2019. Federated learning with personalization layers.
 arXiv preprint arXiv:1912.00818 (2019).</p>
+</blockquote>
 <h1
 id="federated-split-vision-transformer-for-covid-19-cxr-diagnosis-using-task-agnostic-training">Federated
 Split Vision Transformer for COVID-19 CXR Diagnosis using Task-Agnostic
@@ -793,9 +838,11 @@ <h2 id="motivation-5">Motivation</h2>
 <h2 id="method-8">Method</h2>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image70.png" /></p>
+<blockquote>
 <p>Park S, Kim G, Kim J, et al. Federated Split Vision Transformer for
 COVID-19 CXR Diagnosis using Task-Agnostic Training[J]. arXiv preprint
 arXiv:2111.01338, 2021.</p>
+</blockquote>
 <h1
 id="local-learning-matters-rethinking-data-heterogeneity-in-federated-learning">Local
 Learning Matters: Rethinking Data Heterogeneity in Federated
@@ -817,10 +864,14 @@ <h3 id="gradaug-2020">GradAug (2020)</h3>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image73.png" /></p>
 <p>Random number of channels in each layer</p>
+<blockquote>
 <p>University of Central Florida</p>
+</blockquote>
+<blockquote>
 <p>Mendieta M, Yang T, Wang P, et al. Local Learning Matters: Rethinking
 Data Heterogeneity in Federated Learning[J]. arXiv preprint
 arXiv:2111.14213, 2021.</p>
+</blockquote>
 <h1
 id="local-learning-matters-rethinking-data-heterogeneity-in-federated-learning-1">Local
 Learning Matters: Rethinking Data Heterogeneity in Federated
@@ -843,10 +894,14 @@ <h2 id="method-10">Method</h2>
 <h2 id="result-3">Result</h2>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image74.png" /></p>
+<blockquote>
 <p>University of Central Florida</p>
+</blockquote>
+<blockquote>
 <p>Mendieta M, Yang T, Wang P, et al. Local Learning Matters: Rethinking
 Data Heterogeneity in Federated Learning[J]. arXiv preprint
 arXiv:2111.14213, 2021.</p>
+</blockquote>
 <h1 id="motion-representations-for-articulated-animation">Motion
 Representations for Articulated Animation</h1>
 <p>2022.04.15</p>
@@ -874,10 +929,14 @@ <h2 id="问题">问题</h2>
 <h2 id="结果">结果</h2>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image76.gif" /></p>
+<blockquote>
 <p>University of Trento, Italy</p>
+</blockquote>
+<blockquote>
 <p>Siarohin, Aliaksandr, Oliver, Woodford, Jian, Ren, Menglei, Chai, and
 Sergey, Tulyakov. "Motion Representations for Articulated Animation." .
 In CVPR.2021.</p>
+</blockquote>
 <h1
 id="communication-efficient-federated-learning-via-knowledge-distillation">Communication-efficient
 federated learning via knowledge distillation</h1>
@@ -888,22 +947,29 @@ <h2 id="method-11">Method</h2>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image79.png" /></p>
 <p>SVD energy 随着轮数增加，阈值需要增大，即重要的知识越来越多</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image80.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image82.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image81.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image83.png" /></p>
+<p><span class="math display">\[{\mathcal
+L}_{t,i}^{d}=\frac{\mathrm{KL}({\bf y}_{i}^{s}{\bf
+y}_{i}^{t})}{\mathcal{L}_{t,i}^{t}+\mathcal{L}_{s,i}^{t}}\]</span></p>
+<p><span
+class="math display">\[\mathcal{L}_{t,i}^{t}=\mathrm{CE}(\mathrm{y}_{i},\mathrm{y}_{i}^{t})\]</span></p>
+<p><span
+class="math display">\[\mathcal{L}_{t,i}^{h}=\mathcal{L}_{s,i}^{h}=\frac{\mathrm{MSE}({\mathbf
+H}_i^t,{\mathbf W}_{i}^{h}H^{s})+\mathrm{MSE}({\bf A}_{i}^{t},{\bf
+A}^{s})}{\mathcal{L}_{t,i}^{t}+\mathcal{L}_{s,i}^{t}}\]</span></p>
+<p><span
+class="math display">\[T(t)=T_{\mathrm{start}}+(T_{\mathrm{end}}-T_{\mathrm{start}})t,t\in[0,1]\]</span></p>
 <ol type="1">
-<li><p>Client: Layer SVD-&gt;u<em>sigma</em>v</p></li>
-<li><p>Server: mean(u<em>sigma</em>v)-&gt; u<em>sigma</em>v</p></li>
-<li><p>Client Revice: replace u<em>sigma</em>v</p></li>
+<li><p>Client: Layer SVD-&gt;u*sigma*v</p></li>
+<li><p>Server: mean(u*sigma*v)-&gt; u*sigma*v</p></li>
+<li><p>Client Revice: replace u*sigma*v</p></li>
 </ol>
+<blockquote>
+<p>Tsinghua University</p>
+</blockquote>
+<blockquote>
 <p>Wu, C., Wu, F., Lyu, L. et al. Communication-efficient federated
 learning via knowledge distillation. Nat Commun 13, 2032 (2022)</p>
-<p>Tsinghua University</p>
+</blockquote>
 <h1
 id="personalized-federated-learning-using-hypernetworks">Personalized
 Federated Learning using Hypernetworks</h1>
@@ -920,9 +986,11 @@ <h2 id="method-12">Method</h2>
 <h2 id="results">Results</h2>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image86.png" /></p>
+<blockquote>
 <p>Shamsian A, Navon A, Fetaya E, et al. Personalized federated learning
 using hypernetworks[C]//International Conference on Machine Learning.
 PMLR, 2021: 9489-9502.</p>
+</blockquote>
 <h1 id="the-power-of-scale-for-parameter-efficient-prompt-tuning">The
 Power of Scale for Parameter-Efficient Prompt Tuning</h1>
 <p>2023.04.04</p>
@@ -943,9 +1011,13 @@ <h2 id="method-13">Method</h2>
 <h2 id="result-4">Result</h2>
 <p><img
 src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image92.png" /></p>
+<blockquote>
+<p>Google</p>
+</blockquote>
+<blockquote>
 <p>Lester B , et al. The Power of Scale for Parameter-Efficient Prompt
 Tuning. In EMNLP. 2021:3045–3059</p>
-<p>Google</p>
+</blockquote>
 
     </div>
 
diff --git a/css/main.css b/css/main.css
index 1d7fdccd..4b9d821f 100644
--- a/css/main.css
+++ b/css/main.css
@@ -1165,7 +1165,7 @@ pre .javascript .function {
 }
 .links-of-author a::before,
 .links-of-author span.exturl::before {
-  background: #ffff04;
+  background: #ff0fcc;
   border-radius: 50%;
   content: ' ';
   display: inline-block;
diff --git a/imgs/FL_papers/image80.png b/imgs/FL_papers/image80.png
deleted file mode 100644
index 4d2bde4d..00000000
Binary files a/imgs/FL_papers/image80.png and /dev/null differ
diff --git a/imgs/FL_papers/image81.png b/imgs/FL_papers/image81.png
deleted file mode 100644
index 854fd123..00000000
Binary files a/imgs/FL_papers/image81.png and /dev/null differ
diff --git a/imgs/FL_papers/image82.png b/imgs/FL_papers/image82.png
deleted file mode 100644
index 912039e0..00000000
Binary files a/imgs/FL_papers/image82.png and /dev/null differ
diff --git a/imgs/FL_papers/image83.png b/imgs/FL_papers/image83.png
deleted file mode 100644
index 8f2847a1..00000000
Binary files a/imgs/FL_papers/image83.png and /dev/null differ
diff --git a/page/3/index.html b/page/3/index.html
index 89857ed0..6ffb61d3 100644
--- a/page/3/index.html
+++ b/page/3/index.html
@@ -391,7 +391,7 @@ <h2 class="post-title" itemprop="name headline">
                   <i class="far fa-calendar-check"></i>
                 </span>
                 <span class="post-meta-item-text">更新于</span>
-                <time title="عُدل：2024-09-14 21:40:27" itemprop="dateModified" datetime="2024-09-14T21:40:27+08:00">2024-09-14</time>
+                <time title="عُدل：2024-09-14 22:11:05" itemprop="dateModified" datetime="2024-09-14T22:11:05+08:00">2024-09-14</time>
               </span>
             <span class="post-meta-item">
               <span class="post-meta-item-icon">
diff --git a/search.xml b/search.xml
index 6c6d07e8..a77333ea 100644
--- a/search.xml
+++ b/search.xml
@@ -1,581 +1,40 @@
 <?xml version="1.0" encoding="utf-8"?>
 <search>
   <entry>
-    <title>iCalendar：ICS在线课表制作</title>
-    <url>/hexo_blog/2020/09/16/ICS%E5%9C%A8%E7%BA%BF%E8%AF%BE%E8%A1%A8%E5%88%B6%E4%BD%9C/</url>
-    <content><![CDATA[<p>如何制作在线的日历ics文件，最后介绍各个设备如何使用</p>
-<span id="more"></span>
-<h2 id="是什么">是什么</h2>
-<p>ICS为iCalendar的文件名，它是“日历数据交换”的标准。大多数日历本质创建事项都是通过ics这种格式来生成、解析。详见<a
-href="https://zh.wikipedia.org/wiki/ICalendar">Wikipedia</a></p>
-<h2 id="为什么">为什么</h2>
-<p>一般来说，对于临时会议/事件，就直接在日历添加事件即可。但如果是一些周期性、某个时间段持续的事件活动，如果手动添加就很麻烦，比如最常见的就是课表。还有一些如学术会议、球赛、定期活动、节假日等。这类往往是通过他人制作好ics文件，本地导入即可。</p>
-<h2 id="怎么做">怎么做</h2>
-<p>我的目标就是制作自己的课表（在线）</p>
-<h3 id="格式">格式</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">BEGIN:VCALENDAR</span><br><span class="line">VERSION:2.0</span><br><span class="line">PRODID:-//hacksw/handcal//NONSGML v1.0//EN</span><br><span class="line">BEGIN:VEVENT</span><br><span class="line">UID:uid1@example.com</span><br><span class="line">DTSTAMP:19970714T170000Z</span><br><span class="line">ORGANIZER;CN=John Doe:MAILTO:john.doe@example.com</span><br><span class="line">DTSTART:19970714T170000Z</span><br><span class="line">DTEND:19970715T035959Z</span><br><span class="line">SUMMARY:Bastille Day Party</span><br><span class="line">END:VEVENT</span><br><span class="line">END:VCALENDAR</span><br></pre></td></tr></table></figure>
-<p>以行为分隔，每行以冒号分隔。</p>
-<p>第一行，每个ics文件的开头，对应最后一行</p>
-<p>第二行，版本（默认）</p>
-<p>第三行：可自定义，文件说明</p>
-<p>第四行：定义一个事件名称，对应倒数第二行</p>
-<p>第五行：唯一的id</p>
-<p>第六行：创建这个事件的时间</p>
-<p>第七行：可忽略</p>
-<p>第八行：开始时间</p>
-<p>第九行：结束时间</p>
-<p>第十行：事件名称</p>
-<h3 id="我的示例">我的示例</h3>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">BEGIN:VCALENDAR</span><br><span class="line">VERSION:2.0</span><br><span class="line">PRODID:-//JH-L//JH-L Calendar//</span><br><span class="line">BEGIN:VEVENT</span><br><span class="line">SUMMARY:课程A</span><br><span class="line">DTSTART;TZID=&quot;UTC+08:00&quot;;VALUE=DATE-TIME:20200914T083000</span><br><span class="line">DTEND;TZID=&quot;UTC+08:00&quot;;VALUE=DATE-TIME:20200914T101000</span><br><span class="line">DTSTAMP;VALUE=DATE-TIME:20200915T000946Z</span><br><span class="line">UID:c39b3578-f6e7-11ea-bd58-525400eb2034</span><br><span class="line">DESCRIPTION:老师A</span><br><span class="line">LOCATION:上课地点B</span><br><span class="line">END:VEVENT</span><br><span class="line">BEGIN:VEVENT</span><br><span class="line">SUMMARY:课程B</span><br><span class="line">DTSTART;TZID=&quot;UTC+08:00&quot;;VALUE=DATE-TIME:20200916T083000</span><br><span class="line">DTEND;TZID=&quot;UTC+08:00&quot;;VALUE=DATE-TIME:20200916T101000</span><br><span class="line">DTSTAMP;VALUE=DATE-TIME:20200915T000946Z</span><br><span class="line">UID:c39b3af0-f6e7-11ea-bd58-525400eb2034</span><br><span class="line">DESCRIPTION:老师B</span><br><span class="line">LOCATION:上课地点A</span><br><span class="line">END:VEVENT</span><br></pre></td></tr></table></figure>
-<p>相比于原始格式，多添加了一个事件，并且添加了地点
-<code>LOCATION</code>和事件描述 <code>DESCRIPTION</code></p>
-<h3 id="生成">生成</h3>
-<p>既然知道格式，那么生成就很简单了，可以自己按照这种格式手写进一个文档，后缀名为ics即可。这里我借助了Python的icalendar来完成这个工作。</p>
-<p>以下是一个函数，借助这个函数可以生成一个事件，生成后可以添加至日历中。这里uid因为是要求唯一的，所以借助了uuid来完成。这里有一个小坑，由于我们要是北京时间，所以添加时间是，需要加入
-<code>tz_utc_8</code>这个变量
-<code>tz_utc_8 = timezone(timedelta(hours=8))</code></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">cread_event</span>(<span class="params">lesson_name, classroom, teacher, start, end</span>):</span><br><span class="line">    <span class="comment"># 创建事件/日程</span></span><br><span class="line">    event = Event()</span><br><span class="line">    event.add(<span class="string">&#x27;summary&#x27;</span>, lesson_name)</span><br><span class="line"></span><br><span class="line">    dt_now = datetime.now(tz=tz_utc_8)</span><br><span class="line">    event.add(<span class="string">&#x27;dtstart&#x27;</span>, start)</span><br><span class="line">    event.add(<span class="string">&#x27;dtend&#x27;</span>, end)</span><br><span class="line">    <span class="comment"># 创建时间</span></span><br><span class="line">    event.add(<span class="string">&#x27;dtstamp&#x27;</span>, dt_now)</span><br><span class="line">    event.add(<span class="string">&#x27;LOCATION&#x27;</span>, classroom)</span><br><span class="line">    event.add(<span class="string">&#x27;DESCRIPTION&#x27;</span>, <span class="string">&#x27;教师：&#x27;</span> + teacher)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># UID保证唯一</span></span><br><span class="line">    event[<span class="string">&#x27;uid&#x27;</span>] = <span class="built_in">str</span>(uuid.uuid1()) + <span class="string">&#x27;/wnma3mz@gmail.com&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> event</span><br></pre></td></tr></table></figure>
-<p>下一个问题就变为，开始时间与结束时间该怎么写。这里需要使用datetime函数，利用datetime来生成时间格式。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> lesson <span class="keyword">in</span> cls_lst:</span><br><span class="line">    <span class="comment"># 课程名字，教师，教室</span></span><br><span class="line">    <span class="comment"># 课程开始时间(s1小时，s2分钟)，课程结束时间(e1小时，e2分钟)</span></span><br><span class="line">    <span class="comment"># name, teacher, room = f&#x27;&#123;lesson[&quot;name&quot;]&#125;-&#123;lesson[&quot;room&quot;]&#125;&#x27;, lesson[&#x27;teacher&#x27;], lesson[&#x27;room&#x27;]</span></span><br><span class="line">    name, teacher, room = lesson[<span class="string">&#x27;name&#x27;</span>], lesson[<span class="string">&#x27;teacher&#x27;</span>], lesson[<span class="string">&#x27;room&#x27;</span>]</span><br><span class="line">    s1, s2 = lesson[<span class="string">&#x27;time&#x27;</span>][<span class="number">0</span>][<span class="number">0</span>]</span><br><span class="line">    e1, e2 = lesson[<span class="string">&#x27;time&#x27;</span>][-<span class="number">1</span>][-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">for</span> week <span class="keyword">in</span> lesson[<span class="string">&#x27;week&#x27;</span>]:</span><br><span class="line">        <span class="comment"># 第N周</span></span><br><span class="line">        week_delta = timedelta(days=(week - <span class="number">1</span>) * <span class="number">7</span>)</span><br><span class="line">        <span class="keyword">for</span> day <span class="keyword">in</span> lesson[<span class="string">&#x27;day&#x27;</span>]:</span><br><span class="line">            <span class="comment"># 周N</span></span><br><span class="line">            day_delta = timedelta(days=(day - <span class="number">1</span>))</span><br><span class="line">            new_date = begin_date + week_delta + day_delta</span><br><span class="line">            <span class="comment"># 上课的年月日</span></span><br><span class="line">            new_year, new_month, new_day = new_date.year, new_date.month, new_date.day</span><br><span class="line">            ymd = [new_year, new_month, new_day]</span><br><span class="line">            <span class="comment"># 课程开始时间和结束时间</span></span><br><span class="line">            start = datetime(*ymd, s1, s2, tzinfo=tz_utc_8)</span><br><span class="line">            end = datetime(*ymd, e1, e2, tzinfo=tz_utc_8)</span><br><span class="line"></span><br><span class="line">            cal.add_component(cread_event(name, room, teacher, start, end))</span><br></pre></td></tr></table></figure>
-<p>这里需要提前约定好格式。学期开始的年月日、上课的时间及课程上课的周数、时间等。ics是支持固定周期添加事件的，但是由于我的课很乱（上课周数不确定、有的课是1-3节，有的课是2-3节，有的课是1-2节），所以我这里没有使用周期这个功能。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">time_dict = &#123;</span><br><span class="line">    <span class="number">1</span>: [(<span class="number">8</span>, <span class="number">30</span>), (<span class="number">9</span>, <span class="number">20</span>)],</span><br><span class="line">    <span class="number">2</span>: [(<span class="number">9</span>, <span class="number">20</span>), (<span class="number">10</span>, <span class="number">10</span>)],</span><br><span class="line">    <span class="number">3</span>: [(<span class="number">10</span>, <span class="number">30</span>), (<span class="number">11</span>, <span class="number">20</span>)],</span><br><span class="line">    <span class="number">4</span>: [(<span class="number">11</span>, <span class="number">20</span>), (<span class="number">12</span>, <span class="number">10</span>)],</span><br><span class="line">    <span class="number">5</span>: [(<span class="number">13</span>, <span class="number">30</span>), (<span class="number">14</span>, <span class="number">20</span>)],</span><br><span class="line">    <span class="number">6</span>: [(<span class="number">14</span>, <span class="number">20</span>), (<span class="number">15</span>, <span class="number">10</span>)],</span><br><span class="line">    <span class="number">7</span>: [(<span class="number">15</span>, <span class="number">30</span>), (<span class="number">16</span>, <span class="number">20</span>)],</span><br><span class="line">    <span class="number">8</span>: [(<span class="number">16</span>, <span class="number">20</span>), (<span class="number">17</span>, <span class="number">10</span>)],</span><br><span class="line">    <span class="number">9</span>: [(<span class="number">18</span>, <span class="number">10</span>), (<span class="number">19</span>, <span class="number">00</span>)],</span><br><span class="line">    <span class="number">10</span>: [(<span class="number">19</span>, <span class="number">00</span>), (<span class="number">19</span>, <span class="number">50</span>)],</span><br><span class="line">    <span class="number">11</span>: [(<span class="number">20</span>, <span class="number">10</span>), (<span class="number">21</span>, <span class="number">00</span>)],</span><br><span class="line">    <span class="number">12</span>: [(<span class="number">21</span>, <span class="number">00</span>), (<span class="number">21</span>, <span class="number">50</span>)],</span><br><span class="line">&#125;</span><br><span class="line">begin_year = <span class="number">2020</span></span><br><span class="line">begin_month = <span class="number">9</span></span><br><span class="line">begin_day = <span class="number">7</span></span><br><span class="line"></span><br><span class="line">cls_lst = [</span><br><span class="line">        &#123;</span><br><span class="line">            <span class="string">&#x27;name&#x27;</span>: <span class="string">&#x27;课程A&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;teacher&#x27;</span>: <span class="string">&#x27;教师名称&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;room&#x27;</span>: <span class="string">&#x27;教室&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;time&#x27;</span>: [time_dict[<span class="number">1</span>], time_dict[<span class="number">2</span>]], <span class="comment"># 第一节课-第二节课</span></span><br><span class="line">            <span class="string">&#x27;week&#x27;</span>: [<span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">9</span>, <span class="number">10</span>], <span class="comment"># 2-10周</span></span><br><span class="line">            <span class="string">&#x27;day&#x27;</span>: [<span class="number">1</span>, <span class="number">3</span>] <span class="comment"># 周一、周三</span></span><br><span class="line">        &#125;,</span><br><span class="line">]</span><br></pre></td></tr></table></figure>
-<p>完整代码见<a
-href="https://github.com/wnma3mz/Tools/blob/master/myics.py">https://github.com/wnma3mz/Tools/blob/master/others/myics.py</a></p>
-<h3 id="部署">部署</h3>
-<p>生成ics文件后，如果不需要更新的话，那么可以直接导入到本地的日历中。IOS系统可直接导入，安卓手机部分不支持，且如果导入后删除可能需要手动一个一个删除。。。</p>
-<p>注：小米、VIVO经测试不可导入，因为厂商阉割了此功能；魅族、荣耀、三星、苹果、华为均可导入，其中苹果导入的方式是用邮件发送到ios设备上的已绑定的邮件中。</p>
-<p>故基于此，最终考虑还是使用在线部署的方式进行导入。为方便IOS导入，所以需要配置好https。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">server &#123;</span><br><span class="line">    listen       port;</span><br><span class="line">    listen       [::]:port ipv6only=on;</span><br><span class="line">    server_name  ics.test.com; <span class="comment"># ios导入必须使用域名</span></span><br><span class="line">    root         /ics_dir;;</span><br><span class="line">    index        my.ics;</span><br><span class="line">    rewrite ^(.*)$ https://$host$<span class="number">1</span> permanent;</span><br><span class="line">  </span><br><span class="line">    location / &#123;</span><br><span class="line">        root  /ics_dir;</span><br><span class="line">        index my.ics;</span><br><span class="line">	    access_log off;</span><br><span class="line">	    expires 1d;</span><br><span class="line">    &#125;</span><br><span class="line">    location ~* ^.+\.(ics) &#123;</span><br><span class="line">        root /ics_dir;</span><br><span class="line"> 	    access_log off;</span><br><span class="line"> 	    expires 1d;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">server &#123;</span><br><span class="line">       listen       <span class="number">443</span> ssl http2;</span><br><span class="line">       listen       [::]:<span class="number">443</span> ssl http2;</span><br><span class="line">       server_name  ics.test.com;</span><br><span class="line">       ssl_certificate /etc/nginx/ics.test.com/Nginx/1_ics.test.com_bundle.crt;</span><br><span class="line">       ssl_certificate_key /etc/nginx/ics.test.com/Nginx/2_ics.test.com.key;</span><br><span class="line">       location / &#123;</span><br><span class="line">           tcp_nodelay on;</span><br><span class="line">           proxy_set_header Host $host;</span><br><span class="line">           proxy_set_header X-Real-IP $remote_addr;</span><br><span class="line">           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;</span><br><span class="line">           root         /ics_dir;</span><br><span class="line">           index        my.ics;</span><br><span class="line">       &#125;</span><br><span class="line">       location ~* ^.+\.(ics) &#123;</span><br><span class="line">	         root /ics_dir;</span><br><span class="line">	         access_log off;</span><br><span class="line">		       expires 1d;</span><br><span class="line">       &#125;</span><br><span class="line">     &#125;</span><br></pre></td></tr></table></figure>
-<p>部署成功后，直接访问ics.test.com会直接该ics文件</p>
-<h3 id="各个设备配置">各个设备配置</h3>
-<p>Windows端（浏览器）：Outlook、谷歌日历均有在线导入ics文件的入口</p>
-<p>IOS/MacOS端：添加账号中，导入ics文件的域名即可</p>
-<p>安卓端：由于原生日历不支持该项功能，故使用OneCalendar。当然，也可以在Windows端配置好后，下载对应APP亦可。</p>
-<h2 id="写到最后">写到最后</h2>
-<p>制作完成之后发现，我校居然自动生成了ics文件。。。（但感觉还是不如自己做的好。。。。）</p>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>ICS</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>LLM 不同硬件推理速度对比</title>
-    <url>/hexo_blog/2024/07/21/LLM%20%E7%A1%AC%E4%BB%B6%E6%AF%94%E4%BB%B7/</url>
-    <content><![CDATA[<p>对比不同硬件下，LLM 的推理速度</p>
+    <title>Knowledge Distillation in LLM</title>
+    <url>/hexo_blog/2024/08/08/Knowledge-Distillation-in-LLM/</url>
+    <content><![CDATA[<p>从知识蒸馏的概念出发，介绍 LLM 中的知识蒸馏</p>
 <span id="more"></span>
-<p>生成阶段，20 Token/s 认为是一个比较可以接受的速度，即 20 / 1000 =
-0.02 s = 20 ms</p>
-<table style="width:100%;">
-<colgroup>
-<col style="width: 16%" />
-<col style="width: 16%" />
-<col style="width: 16%" />
-<col style="width: 16%" />
-<col style="width: 16%" />
-<col style="width: 16%" />
-</colgroup>
-<thead>
-<tr>
-<th>设备</th>
-<th>模型尺寸</th>
-<th>速度</th>
-<th>来源</th>
-<th>记录时间</th>
-<th>备注</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>1 x RasPi 5 8 GB</td>
-<td>Llama 3 8B （Q4）</td>
-<td>564.31 ms, 1.77 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>I: 556.67 ms, T: 6.17 ms</td>
-<td>https://github.com/b4rtaz/distributed-llama/blob/main/README.md</td>
-<td>2024.07.21</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>4 x RasPi 5 8 GB</td>
-<td>Llama 3 8B （Q4）</td>
-<td>331.47 ms, 3.01 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>I: 267.62 ms, T: 62.34 ms</td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>8 x RasPi 4B 8 GB</td>
-<td>Llama 2 70B（Q4）</td>
-<td>4842.81 ms</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>I: 2121.94 ms, T: 2719.62 ms</td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>c3d-highcpu-30</td>
-<td>Llama 2 7B （Q4）</td>
-<td>101.81 ms</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>I: 101.06 ms, T: 0.19 ms</td>
-<td>https://github.com/b4rtaz/distributed-llama/discussions/9</td>
-<td></td>
-<td>(30 vCPU, 15 core, 59 GB memory) europe-west1, AMD Genoa</td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>c3d-highcpu-30 *4</td>
-<td>Llama 2 7B（Q4）</td>
-<td>53.69 ms</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>I: 40.25 ms, T: 12.81 ms</td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>c3d-highcpu-30</td>
-<td>Llama 2 70B（Q4）</td>
-<td>909.69 ms</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>I: 907.25 ms, T: 1.75 ms</td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>c3d-highcpu-30 *4</td>
-<td>Llama 2 70B（Q4）</td>
-<td>293.06 ms</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>I: 264.00 ms, T: 28.50 ms</td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M1</td>
-<td>Llama 7B（Q4）</td>
-<td>14.19 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/discussions/4167</td>
-<td>2023.11.22</td>
-<td>取最慢的速度</td>
-</tr>
-<tr>
-<td>M1 Pro</td>
-<td>Llama 7B（Q4）</td>
-<td>35.52 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M1 Max</td>
-<td>Llama 7B（Q4）</td>
-<td>54.61 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M1 Ultra</td>
-<td>Llama 7B（Q4）</td>
-<td>74.93 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M2</td>
-<td>Llama 7B（Q4）</td>
-<td>21.7 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M2 Pro</td>
-<td>Llama 7B（Q4）</td>
-<td>37.87 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M2 Max</td>
-<td>Llama 7B（Q4）</td>
-<td>60.99 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M2 Ultra</td>
-<td>Llama 7B（Q4）</td>
-<td>65.95 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M3</td>
-<td>Llama 7B（Q4）</td>
-<td>21.34 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M3 Pro</td>
-<td>Llama 7B（Q4）</td>
-<td>30.65 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>M3 Max</td>
-<td>Llama 7B（Q4）</td>
-<td>56.58 t/s</td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>AMD EPYC 7443P</td>
-<td>Llama 7B（Q4）</td>
-<td>11.18 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1465138574</td>
-<td>2023.3.12</td>
-<td></td>
-</tr>
-<tr>
-<td>Ryzen 7 3700X</td>
-<td>Llama 7B（Q4）</td>
-<td>8.51 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1465313724</td>
-<td>2023.3.13</td>
-<td></td>
-</tr>
-<tr>
-<td>13900k</td>
-<td>Llama 7B（Q4）</td>
-<td>14.02 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1467067155</td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td>2x Intel Xeon Gold 5120 @ 2.20GHz</td>
-<td>Llama 7B（Q4）</td>
-<td>8.68 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1471171246</td>
-<td>2023.3.16</td>
-<td></td>
-</tr>
-<tr>
-<td>E5-2680v4</td>
-<td>Llama 7B（Q4）</td>
-<td>8.87 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1517704976</td>
-<td>2023.4.21</td>
-<td></td>
-</tr>
-<tr>
-<td>i5 6500</td>
-<td>Llama 7B（Q4）</td>
-<td>13.82 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1550410400</td>
-<td>2023.5.17</td>
-<td></td>
-</tr>
-<tr>
-<td>Hetzner Cloud Arm64 Ampere, 16 VCPU</td>
-<td>Llama 7B（Q4）</td>
-<td>11.76 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1575736794</td>
-<td>2023.6.5</td>
-<td></td>
-</tr>
-<tr>
-<td>13900k</td>
-<td>Llama 7B（Q4）</td>
-<td>12.65 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1675971336</td>
-<td>2023.8.12</td>
-<td></td>
-</tr>
-<tr>
-<td>Snapdragon 870 / 8GB of ram</td>
-<td>zephyr-7b (Q4)</td>
-<td>4.7 t/s</td>
-<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1825489115</td>
-<td>2023.11.24</td>
-<td></td>
-</tr>
-<tr>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-<tr>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-</tr>
-</tbody>
-</table>
-<p>I ：推理每个 token花费的时间</p>
-<p>T：通信时间</p>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>NLP</tag>
-        <tag>LLM</tag>
-        <tag>Raspberry</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>LLM的推理相关计算公式</title>
-    <url>/hexo_blog/2024/09/05/LLM%E6%8E%A8%E7%90%86%E7%9B%B8%E5%85%B3%E8%AE%A1%E7%AE%97%E5%85%AC%E5%BC%8F/</url>
-    <content><![CDATA[<p>记录 LLM 在推理上的理论计算公式</p>
-<span id="more"></span>
-<h3 id="参数量">参数量</h3>
-<p>embedding + lm head + transformer block <span
-class="math inline">\(\times\)</span> N</p>
-<ul>
-<li>Embedding / LM Head 的参数量：vocab_size <span
-class="math inline">\(\times\)</span> hidden_size</li>
-<li>transformer block 的参数量
-<ul>
-<li>self-attention：qkvo 四个线性层，需要分别考虑两种 Attention
+<h2 id="outlines">Outlines</h2>
 <ul>
-<li>multi-head attention（MHA）：4 <span
-class="math inline">\(\times\)</span> hidden_size <span
-class="math inline">\(^2\)</span></li>
-<li>grouped query attention（GQA）：
+<li>Knowledge Distillation（知识蒸馏）
 <ul>
-<li><p>q 和 o，2 $$ hidden_size <span
-class="math inline">\(^2\)</span></p></li>
-<li><p>k 和 v，2 $$ hidden_size <span class="math inline">\(^2\)</span>
-/ num_attention_heads $$ num_key_value_heads</p></li>
-</ul></li>
-</ul></li>
-<li>FFN：三个线性层，3 <span class="math inline">\(\times\)</span>
-hidden_size <span class="math inline">\(\times\)</span>
-intermediate_size</li>
-<li>layer norm：2 <span class="math inline">\(\times\)</span> 2 <span
-class="math inline">\(\times\)</span> hidden_size</li>
+<li>是什么</li>
+<li>怎么做</li>
 </ul></li>
-</ul>
-<p>以 <a
-href="https://huggingface.co/meta-llama/Llama-2-13b-hf/blob/main/config.json">llama2-13B</a>
-为例，关键参数</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;hidden_size&quot;</span><span class="punctuation">:</span> <span class="number">5120</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;intermediate_size&quot;</span><span class="punctuation">:</span> <span class="number">13824</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;mlp_bias&quot;</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_attention_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_hidden_layers&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_key_value_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;vocab_size&quot;</span><span class="punctuation">:</span> <span class="number">32000</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
-<p>由于 num_attention_heads == num_key_value_heads，所以是 MHA</p>
+<li>LLM 中的 KD 及其变种
 <ul>
-<li>Embedding/LM Head： <span class="math inline">\(32000\times
-5120\)</span></li>
-<li>transformer block
+<li>Reverse KD</li>
+<li>JS 散度</li>
+</ul></li>
+<li>论文
 <ul>
-<li>self-attention：<span class="math inline">\(4\times 5120\times
-5120\)</span></li>
-<li>FFN: <span class="math inline">\(3\times 5120\times
-13824\)</span></li>
-<li>layer norm: <span class="math inline">\(4\times 5120\)</span></li>
+<li>MiniLLM: Knowledge Distillation of Large Language Models</li>
+<li>Revisiting Knowledge Distillation for Autoregressive Language
+Models</li>
 </ul></li>
 </ul>
-<p>综上，列出计算</p>
-<p><span class="math display">\[2\times 32000\times 5120 + 40\times
-(4\times 5120\times 5120 + 3\times 5120\times 13824 + 4\times 5120) =
-13,016,268,800 \approx 13B\]</span></p>
-<p>如果是 (b)float16，则占用空间/显存为 13 $$ 2 = 26G</p>
-<p>如果是 GQA，self-attn 参数量公式为</p>
-<h3 id="kv-cache-显存计算公式">KV cache 显存计算公式</h3>
-<p><span class="math inline">\(4\times b\times l\times num\_heads\times
-embed\_size\_per\_head \times (s+n)\)</span></p>
-<p>参数说明</p>
+<h2 id="知识蒸馏是什么">知识蒸馏是什么</h2>
+<p><strong>蒸馏的作用</strong>：清除绝大部分杂质和杀死微生物</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/kdinLLM/kd1.png" /></p>
+<p><strong>知识</strong>：高度抽象的概念</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/kdinLLM/kd2.png" /></p>
+<p>在深度学习模型里面，将其具象为：</p>
 <ul>
-<li>b: 句子条数</li>
-<li>l：层数</li>
-<li>num_heads：隐层大小 (num_key_value_heads)</li>
-<li>embed_size_per_head：每个头的大小 (hidden_size /
-num_attention_heads)</li>
-<li>s：输入长度</li>
-<li>n：输出长度</li>
-<li>4：k cache+v cache，均为 float16，所以是(1+1)*2</li>
-</ul>
-<p>如果只有1条句子，输入+输出 token 长度由 512 -&gt; 1024，则会增加
-<span class="math inline">\(4\times 512 \times l\times h\)</span>：</p>
-<p>Llama3-8B：$4 / 1024 / 1024 = 64 $ M</p>
-<p>Llama3-70B：<span class="math inline">\(4\times 512\times 1024\times
-80 / 1024 / 1024 = 160\)</span> M</p>
-<p>简单来说，每增加一个 token</p>
-<p>8B 就会增加 0.125 M 的显存</p>
-<p>70B 就会增加 0.3125 M 的显存</p>
-<h3 id="计算量">计算量</h3>
-<p>Embedding 可以视作一个哈希表，没有计算量</p>
-<ul>
-<li>LM Head：<span class="math inline">\(2 \times b \times s\times
-hidden\_size\times V\)</span></li>
-<li>Self-attention：
-<ul>
-<li>q_proj: <span class="math inline">\(2 \times b \times s \times
-hidden\_size^2\)</span></li>
-<li>k_proj和v_proj:
-<ul>
-<li>MHA: <span class="math inline">\(2\times(2 \times b \times s \times
-hidden\_size^2)\)</span></li>
-<li>GQA: <span class="math inline">\(2\times(2 \times b \times s \times
-hidden\_size^2 / (num\_attention\_heads
-\times  num\_key\_value\_heads)^2 )\)</span></li>
-</ul></li>
-<li>attn_weights: <span class="math inline">\(2 \times b \times s^2
-\times hidden\_size\)</span></li>
-<li>attn_output: <span class="math inline">\(2 \times b \times s^2
-\times hidden\_size\)</span></li>
-<li>o_proj: <span class="math inline">\(2 \times b \times s \times
-hidden\_size^2\)</span></li>
-</ul></li>
-<li>FFN:
-<ul>
-<li>gate_proj 和 up_proj: <span class="math inline">\(2\times 2\times b
-\times s\times (hidden\_size\times intermediate\_size)\)</span></li>
-<li>down_proj: <span class="math inline">\(2\times b \times s\times
-(intermediate\_size\times hidden\_size)\)</span></li>
-</ul></li>
-</ul>
-<p>参数说明 - b: 句子条数 - s: 输入长度 - 2:
-一次乘法操作和一次加法操作</p>
-<p>还是以 <a
-href="https://huggingface.co/meta-llama/Llama-2-13b-hf/blob/main/config.json">llama2-13B</a>
-为例，关键参数</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;hidden_size&quot;</span><span class="punctuation">:</span> <span class="number">5120</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;intermediate_size&quot;</span><span class="punctuation">:</span> <span class="number">13824</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;mlp_bias&quot;</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_attention_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_hidden_layers&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_key_value_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;vocab_size&quot;</span><span class="punctuation">:</span> <span class="number">32000</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
-<p>假设 b=1，s=1，则整体计算量有</p>
-<ul>
-<li>LM Head: <span class="math inline">\(2\times 5120\times
-32000\)</span></li>
-<li>self-attention:
-<ul>
-<li><span class="math inline">\(2\times 5120\times 5120\)</span></li>
-<li><span class="math inline">\(4\times 5120\times 5120\)</span></li>
-<li><span class="math inline">\(2\times 5120\)</span></li>
-<li><span class="math inline">\(2\times 5120\)</span></li>
-<li><span class="math inline">\(2\times 5120\times 5120\)</span></li>
-</ul></li>
-<li>MLP:
-<ul>
-<li><span class="math inline">\(4\times 5120\times 13824\)</span></li>
-<li><span class="math inline">\(2\times 13824\times 5120\)</span></li>
-</ul></li>
-</ul>
-<p>模型有 40 层，所以</p>
-<p><span class="math inline">\(2\times 5120\times 32000+
-40\times(8\times 5120\times 5120+4\times 5120+ 2\times 5120\times
-13824+2\times 13824\times 5120) = 20,041,728,000\)</span></p>
-<p>参考资料：</p>
-<p>https://blog.csdn.net/wxc971231/article/details/135434478</p>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>NLP</tag>
-        <tag>Inference</tag>
-        <tag>Math</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Knowledge Distillation in LLM</title>
-    <url>/hexo_blog/2024/08/08/Knowledge-Distillation-in-LLM/</url>
-    <content><![CDATA[<p>从知识蒸馏的概念出发，介绍 LLM 中的知识蒸馏</p>
-<span id="more"></span>
-<h2 id="outlines">Outlines</h2>
-<ul>
-<li>Knowledge Distillation（知识蒸馏）
-<ul>
-<li>是什么</li>
-<li>怎么做</li>
-</ul></li>
-<li>LLM 中的 KD 及其变种
-<ul>
-<li>Reverse KD</li>
-<li>JS 散度</li>
-</ul></li>
-<li>论文
-<ul>
-<li>MiniLLM: Knowledge Distillation of Large Language Models</li>
-<li>Revisiting Knowledge Distillation for Autoregressive Language
-Models</li>
-</ul></li>
-</ul>
-<h2 id="知识蒸馏是什么">知识蒸馏是什么</h2>
-<p><strong>蒸馏的作用</strong>：清除绝大部分杂质和杀死微生物</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/kdinLLM/kd1.png" /></p>
-<p><strong>知识</strong>：高度抽象的概念</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/kdinLLM/kd2.png" /></p>
-<p>在深度学习模型里面，将其具象为：</p>
-<ul>
-<li><p>模型权重：若干固定好的矩阵</p></li>
-<li><p>模型输出：模型对于输入的响应</p></li>
+<li><p>模型权重：若干固定好的矩阵</p></li>
+<li><p>模型输出：模型对于输入的响应</p></li>
 </ul>
 <p>更具体的，对于一个三分类模型，模型最后输出的是 logits。</p>
 <p>如：<span class="math inline">\([0.2, 0.3,
@@ -989,1210 +448,971 @@ one-shot 编码是 [0, 1, 0] 和 [0, 0, 1] 。</p>
       </tags>
   </entry>
   <entry>
-    <title>LLM 深度的”稀疏性“</title>
-    <url>/hexo_blog/2024/03/28/LLM%E6%B7%B1%E5%BA%A6%E7%9A%84%E7%A8%80%E7%96%8F%E6%80%A7/</url>
-    <content><![CDATA[<p>LLM 在宽度上的稀疏性可以通过 MoE
-这种架构来优化。而深度上也具备稀疏性，故而可以进行裁剪。</p>
+    <title>iCalendar：ICS在线课表制作</title>
+    <url>/hexo_blog/2020/09/16/ICS%E5%9C%A8%E7%BA%BF%E8%AF%BE%E8%A1%A8%E5%88%B6%E4%BD%9C/</url>
+    <content><![CDATA[<p>如何制作在线的日历ics文件，最后介绍各个设备如何使用</p>
 <span id="more"></span>
-<h2 id="背景">背景</h2>
-<p>这篇<a
-href="https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens">博客</a>发现到
-GPT-2 在深度上的输出具有相似性。即，第 20 层的输出接入 LM Head 和第 21
-层的输出接入 LM Head 是相似，甚至一致的。</p>
-<p>故而，模型在深度上可能是稀疏的。所以可以按 block
-的维度进行裁剪。如下图所示，</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/image-2024-3-28_18-23-0.png" /></p>
-<p>找到三篇相似的工作</p>
-<ul>
-<li>上交：<a href="https://arxiv.org/html/2402.11187v1">LaCo: Large
-Language Model Pruning via Layer Collapse</a></li>
-<li>Meta：<a href="https://arxiv.org/html/2403.17887">The Unreasonable
-Ineffectiveness of the Deeper Layers</a></li>
-<li>百川：<a href="https://arxiv.org/html/2403.03853v2">ShortGPT: Layers
-in Large Language Models are More Redundant Than You Expect</a></li>
-</ul>
-<p>越靠近模型输出的那些层相似度较高，可以通过删除一些“冗余”层来减小模型参数</p>
-<h2 id="方法的共通性">方法的共通性</h2>
-<ul>
-<li>构建一个“中性”数据集！！！</li>
-<li>观察不同层的输入/输出</li>
-<li>用某种度量距离的方法测量不同层是否相似
-<ul>
-<li>百川：Block Influence</li>
-<li>Meta：角距离</li>
-</ul></li>
-</ul>
-<h2 id="meta-论文结果">Meta 论文结果</h2>
-<h3 id="不同模型具有相似的表现">不同模型具有相似的表现</h3>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/x2.png" /></p>
-<p>注：</p>
-<ul>
-<li>实线（healing）：扔掉一些层之后进行了训练</li>
-<li>虚线（no healing）： → 扔掉这些层后不训练</li>
-<li>实线和虚线扔掉的是相同的层</li>
-<li>横坐标是扔掉的比例</li>
-</ul>
-<p>Q：MMLU/BoolQ 指标具备足够的可信度？</p>
-<p>A：个人感觉没有，但有后续有其他的一些实验支撑</p>
-<p>Q：Qwen 为啥表现不大一样？</p>
-<p>A：论文其他部分进行一定的解释，但与本文的重点相关性不强，所以不展开介绍</p>
-<h3 id="暴力裁剪-vs-提出的方法">暴力裁剪 vs 提出的方法</h3>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/x5.png" /></p>
-<p>有一种直接粗暴的方法就是直接扔掉最后若干层，这样就不需要数据集啥的来选层。</p>
-<p>奇怪的现象就是直接扔掉最后若干层，指标会出现先降低再提升（不训练的情况）。</p>
-<h2 id="百川论文的结论">百川论文的结论</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/baichuan.png" /></p>
-<p><strong>第一层和最后一层很重要！！！</strong></p>
-<h2 id="剪枝比率">剪枝比率</h2>
-<p>利用宽度稀疏性，一种优化架构 MoE
-可以做到降低一半的参数且保持性能不变。</p>
-<p>而在深度稀疏上，这一点是比较难做到的。</p>
-<table>
+<h2 id="是什么">是什么</h2>
+<p>ICS为iCalendar的文件名，它是“日历数据交换”的标准。大多数日历本质创建事项都是通过ics这种格式来生成、解析。详见<a
+href="https://zh.wikipedia.org/wiki/ICalendar">Wikipedia</a></p>
+<h2 id="为什么">为什么</h2>
+<p>一般来说，对于临时会议/事件，就直接在日历添加事件即可。但如果是一些周期性、某个时间段持续的事件活动，如果手动添加就很麻烦，比如最常见的就是课表。还有一些如学术会议、球赛、定期活动、节假日等。这类往往是通过他人制作好ics文件，本地导入即可。</p>
+<h2 id="怎么做">怎么做</h2>
+<p>我的目标就是制作自己的课表（在线）</p>
+<h3 id="格式">格式</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">BEGIN:VCALENDAR</span><br><span class="line">VERSION:2.0</span><br><span class="line">PRODID:-//hacksw/handcal//NONSGML v1.0//EN</span><br><span class="line">BEGIN:VEVENT</span><br><span class="line">UID:uid1@example.com</span><br><span class="line">DTSTAMP:19970714T170000Z</span><br><span class="line">ORGANIZER;CN=John Doe:MAILTO:john.doe@example.com</span><br><span class="line">DTSTART:19970714T170000Z</span><br><span class="line">DTEND:19970715T035959Z</span><br><span class="line">SUMMARY:Bastille Day Party</span><br><span class="line">END:VEVENT</span><br><span class="line">END:VCALENDAR</span><br></pre></td></tr></table></figure>
+<p>以行为分隔，每行以冒号分隔。</p>
+<p>第一行，每个ics文件的开头，对应最后一行</p>
+<p>第二行，版本（默认）</p>
+<p>第三行：可自定义，文件说明</p>
+<p>第四行：定义一个事件名称，对应倒数第二行</p>
+<p>第五行：唯一的id</p>
+<p>第六行：创建这个事件的时间</p>
+<p>第七行：可忽略</p>
+<p>第八行：开始时间</p>
+<p>第九行：结束时间</p>
+<p>第十行：事件名称</p>
+<h3 id="我的示例">我的示例</h3>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">BEGIN:VCALENDAR</span><br><span class="line">VERSION:2.0</span><br><span class="line">PRODID:-//JH-L//JH-L Calendar//</span><br><span class="line">BEGIN:VEVENT</span><br><span class="line">SUMMARY:课程A</span><br><span class="line">DTSTART;TZID=&quot;UTC+08:00&quot;;VALUE=DATE-TIME:20200914T083000</span><br><span class="line">DTEND;TZID=&quot;UTC+08:00&quot;;VALUE=DATE-TIME:20200914T101000</span><br><span class="line">DTSTAMP;VALUE=DATE-TIME:20200915T000946Z</span><br><span class="line">UID:c39b3578-f6e7-11ea-bd58-525400eb2034</span><br><span class="line">DESCRIPTION:老师A</span><br><span class="line">LOCATION:上课地点B</span><br><span class="line">END:VEVENT</span><br><span class="line">BEGIN:VEVENT</span><br><span class="line">SUMMARY:课程B</span><br><span class="line">DTSTART;TZID=&quot;UTC+08:00&quot;;VALUE=DATE-TIME:20200916T083000</span><br><span class="line">DTEND;TZID=&quot;UTC+08:00&quot;;VALUE=DATE-TIME:20200916T101000</span><br><span class="line">DTSTAMP;VALUE=DATE-TIME:20200915T000946Z</span><br><span class="line">UID:c39b3af0-f6e7-11ea-bd58-525400eb2034</span><br><span class="line">DESCRIPTION:老师B</span><br><span class="line">LOCATION:上课地点A</span><br><span class="line">END:VEVENT</span><br></pre></td></tr></table></figure>
+<p>相比于原始格式，多添加了一个事件，并且添加了地点
+<code>LOCATION</code>和事件描述 <code>DESCRIPTION</code></p>
+<h3 id="生成">生成</h3>
+<p>既然知道格式，那么生成就很简单了，可以自己按照这种格式手写进一个文档，后缀名为ics即可。这里我借助了Python的icalendar来完成这个工作。</p>
+<p>以下是一个函数，借助这个函数可以生成一个事件，生成后可以添加至日历中。这里uid因为是要求唯一的，所以借助了uuid来完成。这里有一个小坑，由于我们要是北京时间，所以添加时间是，需要加入
+<code>tz_utc_8</code>这个变量
+<code>tz_utc_8 = timezone(timedelta(hours=8))</code></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">cread_event</span>(<span class="params">lesson_name, classroom, teacher, start, end</span>):</span><br><span class="line">    <span class="comment"># 创建事件/日程</span></span><br><span class="line">    event = Event()</span><br><span class="line">    event.add(<span class="string">&#x27;summary&#x27;</span>, lesson_name)</span><br><span class="line"></span><br><span class="line">    dt_now = datetime.now(tz=tz_utc_8)</span><br><span class="line">    event.add(<span class="string">&#x27;dtstart&#x27;</span>, start)</span><br><span class="line">    event.add(<span class="string">&#x27;dtend&#x27;</span>, end)</span><br><span class="line">    <span class="comment"># 创建时间</span></span><br><span class="line">    event.add(<span class="string">&#x27;dtstamp&#x27;</span>, dt_now)</span><br><span class="line">    event.add(<span class="string">&#x27;LOCATION&#x27;</span>, classroom)</span><br><span class="line">    event.add(<span class="string">&#x27;DESCRIPTION&#x27;</span>, <span class="string">&#x27;教师：&#x27;</span> + teacher)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># UID保证唯一</span></span><br><span class="line">    event[<span class="string">&#x27;uid&#x27;</span>] = <span class="built_in">str</span>(uuid.uuid1()) + <span class="string">&#x27;/wnma3mz@gmail.com&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> event</span><br></pre></td></tr></table></figure>
+<p>下一个问题就变为，开始时间与结束时间该怎么写。这里需要使用datetime函数，利用datetime来生成时间格式。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> lesson <span class="keyword">in</span> cls_lst:</span><br><span class="line">    <span class="comment"># 课程名字，教师，教室</span></span><br><span class="line">    <span class="comment"># 课程开始时间(s1小时，s2分钟)，课程结束时间(e1小时，e2分钟)</span></span><br><span class="line">    <span class="comment"># name, teacher, room = f&#x27;&#123;lesson[&quot;name&quot;]&#125;-&#123;lesson[&quot;room&quot;]&#125;&#x27;, lesson[&#x27;teacher&#x27;], lesson[&#x27;room&#x27;]</span></span><br><span class="line">    name, teacher, room = lesson[<span class="string">&#x27;name&#x27;</span>], lesson[<span class="string">&#x27;teacher&#x27;</span>], lesson[<span class="string">&#x27;room&#x27;</span>]</span><br><span class="line">    s1, s2 = lesson[<span class="string">&#x27;time&#x27;</span>][<span class="number">0</span>][<span class="number">0</span>]</span><br><span class="line">    e1, e2 = lesson[<span class="string">&#x27;time&#x27;</span>][-<span class="number">1</span>][-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">for</span> week <span class="keyword">in</span> lesson[<span class="string">&#x27;week&#x27;</span>]:</span><br><span class="line">        <span class="comment"># 第N周</span></span><br><span class="line">        week_delta = timedelta(days=(week - <span class="number">1</span>) * <span class="number">7</span>)</span><br><span class="line">        <span class="keyword">for</span> day <span class="keyword">in</span> lesson[<span class="string">&#x27;day&#x27;</span>]:</span><br><span class="line">            <span class="comment"># 周N</span></span><br><span class="line">            day_delta = timedelta(days=(day - <span class="number">1</span>))</span><br><span class="line">            new_date = begin_date + week_delta + day_delta</span><br><span class="line">            <span class="comment"># 上课的年月日</span></span><br><span class="line">            new_year, new_month, new_day = new_date.year, new_date.month, new_date.day</span><br><span class="line">            ymd = [new_year, new_month, new_day]</span><br><span class="line">            <span class="comment"># 课程开始时间和结束时间</span></span><br><span class="line">            start = datetime(*ymd, s1, s2, tzinfo=tz_utc_8)</span><br><span class="line">            end = datetime(*ymd, e1, e2, tzinfo=tz_utc_8)</span><br><span class="line"></span><br><span class="line">            cal.add_component(cread_event(name, room, teacher, start, end))</span><br></pre></td></tr></table></figure>
+<p>这里需要提前约定好格式。学期开始的年月日、上课的时间及课程上课的周数、时间等。ics是支持固定周期添加事件的，但是由于我的课很乱（上课周数不确定、有的课是1-3节，有的课是2-3节，有的课是1-2节），所以我这里没有使用周期这个功能。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">time_dict = &#123;</span><br><span class="line">    <span class="number">1</span>: [(<span class="number">8</span>, <span class="number">30</span>), (<span class="number">9</span>, <span class="number">20</span>)],</span><br><span class="line">    <span class="number">2</span>: [(<span class="number">9</span>, <span class="number">20</span>), (<span class="number">10</span>, <span class="number">10</span>)],</span><br><span class="line">    <span class="number">3</span>: [(<span class="number">10</span>, <span class="number">30</span>), (<span class="number">11</span>, <span class="number">20</span>)],</span><br><span class="line">    <span class="number">4</span>: [(<span class="number">11</span>, <span class="number">20</span>), (<span class="number">12</span>, <span class="number">10</span>)],</span><br><span class="line">    <span class="number">5</span>: [(<span class="number">13</span>, <span class="number">30</span>), (<span class="number">14</span>, <span class="number">20</span>)],</span><br><span class="line">    <span class="number">6</span>: [(<span class="number">14</span>, <span class="number">20</span>), (<span class="number">15</span>, <span class="number">10</span>)],</span><br><span class="line">    <span class="number">7</span>: [(<span class="number">15</span>, <span class="number">30</span>), (<span class="number">16</span>, <span class="number">20</span>)],</span><br><span class="line">    <span class="number">8</span>: [(<span class="number">16</span>, <span class="number">20</span>), (<span class="number">17</span>, <span class="number">10</span>)],</span><br><span class="line">    <span class="number">9</span>: [(<span class="number">18</span>, <span class="number">10</span>), (<span class="number">19</span>, <span class="number">00</span>)],</span><br><span class="line">    <span class="number">10</span>: [(<span class="number">19</span>, <span class="number">00</span>), (<span class="number">19</span>, <span class="number">50</span>)],</span><br><span class="line">    <span class="number">11</span>: [(<span class="number">20</span>, <span class="number">10</span>), (<span class="number">21</span>, <span class="number">00</span>)],</span><br><span class="line">    <span class="number">12</span>: [(<span class="number">21</span>, <span class="number">00</span>), (<span class="number">21</span>, <span class="number">50</span>)],</span><br><span class="line">&#125;</span><br><span class="line">begin_year = <span class="number">2020</span></span><br><span class="line">begin_month = <span class="number">9</span></span><br><span class="line">begin_day = <span class="number">7</span></span><br><span class="line"></span><br><span class="line">cls_lst = [</span><br><span class="line">        &#123;</span><br><span class="line">            <span class="string">&#x27;name&#x27;</span>: <span class="string">&#x27;课程A&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;teacher&#x27;</span>: <span class="string">&#x27;教师名称&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;room&#x27;</span>: <span class="string">&#x27;教室&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;time&#x27;</span>: [time_dict[<span class="number">1</span>], time_dict[<span class="number">2</span>]], <span class="comment"># 第一节课-第二节课</span></span><br><span class="line">            <span class="string">&#x27;week&#x27;</span>: [<span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">9</span>, <span class="number">10</span>], <span class="comment"># 2-10周</span></span><br><span class="line">            <span class="string">&#x27;day&#x27;</span>: [<span class="number">1</span>, <span class="number">3</span>] <span class="comment"># 周一、周三</span></span><br><span class="line">        &#125;,</span><br><span class="line">]</span><br></pre></td></tr></table></figure>
+<p>完整代码见<a
+href="https://github.com/wnma3mz/Tools/blob/master/myics.py">https://github.com/wnma3mz/Tools/blob/master/others/myics.py</a></p>
+<h3 id="部署">部署</h3>
+<p>生成ics文件后，如果不需要更新的话，那么可以直接导入到本地的日历中。IOS系统可直接导入，安卓手机部分不支持，且如果导入后删除可能需要手动一个一个删除。。。</p>
+<p>注：小米、VIVO经测试不可导入，因为厂商阉割了此功能；魅族、荣耀、三星、苹果、华为均可导入，其中苹果导入的方式是用邮件发送到ios设备上的已绑定的邮件中。</p>
+<p>故基于此，最终考虑还是使用在线部署的方式进行导入。为方便IOS导入，所以需要配置好https。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">server &#123;</span><br><span class="line">    listen       port;</span><br><span class="line">    listen       [::]:port ipv6only=on;</span><br><span class="line">    server_name  ics.test.com; <span class="comment"># ios导入必须使用域名</span></span><br><span class="line">    root         /ics_dir;;</span><br><span class="line">    index        my.ics;</span><br><span class="line">    rewrite ^(.*)$ https://$host$<span class="number">1</span> permanent;</span><br><span class="line">  </span><br><span class="line">    location / &#123;</span><br><span class="line">        root  /ics_dir;</span><br><span class="line">        index my.ics;</span><br><span class="line">	    access_log off;</span><br><span class="line">	    expires 1d;</span><br><span class="line">    &#125;</span><br><span class="line">    location ~* ^.+\.(ics) &#123;</span><br><span class="line">        root /ics_dir;</span><br><span class="line"> 	    access_log off;</span><br><span class="line"> 	    expires 1d;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">server &#123;</span><br><span class="line">       listen       <span class="number">443</span> ssl http2;</span><br><span class="line">       listen       [::]:<span class="number">443</span> ssl http2;</span><br><span class="line">       server_name  ics.test.com;</span><br><span class="line">       ssl_certificate /etc/nginx/ics.test.com/Nginx/1_ics.test.com_bundle.crt;</span><br><span class="line">       ssl_certificate_key /etc/nginx/ics.test.com/Nginx/2_ics.test.com.key;</span><br><span class="line">       location / &#123;</span><br><span class="line">           tcp_nodelay on;</span><br><span class="line">           proxy_set_header Host $host;</span><br><span class="line">           proxy_set_header X-Real-IP $remote_addr;</span><br><span class="line">           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;</span><br><span class="line">           root         /ics_dir;</span><br><span class="line">           index        my.ics;</span><br><span class="line">       &#125;</span><br><span class="line">       location ~* ^.+\.(ics) &#123;</span><br><span class="line">	         root /ics_dir;</span><br><span class="line">	         access_log off;</span><br><span class="line">		       expires 1d;</span><br><span class="line">       &#125;</span><br><span class="line">     &#125;</span><br></pre></td></tr></table></figure>
+<p>部署成功后，直接访问ics.test.com会直接该ics文件</p>
+<h3 id="各个设备配置">各个设备配置</h3>
+<p>Windows端（浏览器）：Outlook、谷歌日历均有在线导入ics文件的入口</p>
+<p>IOS/MacOS端：添加账号中，导入ics文件的域名即可</p>
+<p>安卓端：由于原生日历不支持该项功能，故使用OneCalendar。当然，也可以在Windows端配置好后，下载对应APP亦可。</p>
+<h2 id="写到最后">写到最后</h2>
+<p>制作完成之后发现，我校居然自动生成了ics文件。。。（但感觉还是不如自己做的好。。。。）</p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>ICS</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>LLM 不同硬件推理速度对比</title>
+    <url>/hexo_blog/2024/07/21/LLM%20%E7%A1%AC%E4%BB%B6%E6%AF%94%E4%BB%B7/</url>
+    <content><![CDATA[<p>对比不同硬件下，LLM 的推理速度</p>
+<span id="more"></span>
+<p>生成阶段，20 Token/s 认为是一个比较可以接受的速度，即 20 / 1000 =
+0.02 s = 20 ms</p>
+<table style="width:100%;">
+<colgroup>
+<col style="width: 16%" />
+<col style="width: 16%" />
+<col style="width: 16%" />
+<col style="width: 16%" />
+<col style="width: 16%" />
+<col style="width: 16%" />
+</colgroup>
 <thead>
 <tr>
+<th>设备</th>
 <th>模型尺寸</th>
-<th>层数</th>
-<th>参数量</th>
+<th>速度</th>
+<th>来源</th>
+<th>记录时间</th>
+<th>备注</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td>70B</td>
-<td>80</td>
-<td>69,434,351,616</td>
-</tr>
-<tr>
-<td>70B</td>
-<td>40</td>
-<td>35,208,175,616</td>
+<td>1 x RasPi 5 8 GB</td>
+<td>Llama 3 8B （Q4）</td>
+<td>564.31 ms, 1.77 t/s</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>70B</td>
-<td>45</td>
-<td>39,486,447,616</td>
+<td>I: 556.67 ms, T: 6.17 ms</td>
+<td>https://github.com/b4rtaz/distributed-llama/blob/main/README.md</td>
+<td>2024.07.21</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>34B</td>
-<td>48</td>
-<td>34,201,673,728</td>
+<td>4 x RasPi 5 8 GB</td>
+<td>Llama 3 8B （Q4）</td>
+<td>331.47 ms, 3.01 t/s</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>34B</td>
-<td>24</td>
-<td>17,591,836,672</td>
+<td>I: 267.62 ms, T: 62.34 ms</td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
-</tbody>
-</table>
-<p>简单来说，对于 70B 模型要剪枝到 34B，需要砍掉一半的层；对于 34B
-模型要剪枝到 13B，要砍掉大于一半的层。</p>
-<p>通过 Meta 论文的实验结果显示，当裁剪超过
-40%，就会有较大的风险不能恢复模型性能。</p>
-<h2 id="实验">实验</h2>
-<p>在 boolq/gsm8k
-数据集上简单了进行若干实验，这两个数据集均具备训练集和评测集。所以，对
-13B 模型进行暴力裁剪后（～40%
-参数）在训练集上进行训练，在评测集上进行评测。</p>
-<ul>
-<li>裁剪方法：保留最后一层，从最后一层往前数，直至裁剪到 40% 为止。</li>
-<li>完整训练集：对应评测的训练集+通用的QA/对话数据集</li>
-</ul>
-<h3 id="gsm8k">gsm8k</h3>
-<p>是一个考验数学/逻辑推理能力的评测结果</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;conversations&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;user&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;assistant&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Natalia sold 48 clips in April.\nThen she sold half as many in May, which is 48 / 2 = 24.\nSo altogether she sold 48 + 24 = 72 clips.\nThe answer is 72.&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">]</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
-<p>在该评测上指标直接崩掉，如原来可能是 &gt;70，现在只有 2%</p>
-<table>
-<colgroup>
-<col style="width: 47%" />
-<col style="width: 52%" />
-</colgroup>
-<thead>
 <tr>
-<th><strong>prompt</strong></th>
-<th><strong>pred</strong></th>
+<td>8 x RasPi 4B 8 GB</td>
+<td>Llama 2 70B（Q4）</td>
+<td>4842.81 ms</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
-</thead>
-<tbody>
 <tr>
-<td>Janet’s ducks lay 16 eggs per day. She eats three for breakfast
-every morning and bakes muffins for her friends every day with four. She
-sells the remainder at the farmers' market daily for $2 per fresh duck
-egg. How much in dollars does she make every day at the farmers' market?
-Let's think step by step.</td>
-<td>Each egg is 16 eggs/day for 16 eggs/day for 1600000000 / 16 =
-128000000. So, the total number of eggs is 128000000. The total number
-of eggs is 160000000 - 128000000 = 128000000.The answer is
-128000000.</td>
+<td>I: 2121.94 ms, T: 2719.62 ms</td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>A robe takes 2 bolts of blue fiber and half that much white fiber.
-How many bolts in total does it take? Let's think step by step</td>
-<td>The rose is 2 bolts of blue fibers. Half that amount is 2 / 2 = 1.
-So the rose is 2 - 1 = 2 flobs. The answer is 2.</td>
+<td>c3d-highcpu-30</td>
+<td>Llama 2 7B （Q4）</td>
+<td>101.81 ms</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>Josh decides to try flipping a house. He buys a house for $80,000
-and then puts in $50,000 in repairs. This increased the value of the
-house by 150%. How much profit did he make? Let's think step by
-step.</td>
-<td>The house was originally 80,000 dollars. The new house is 80,000 +
-50,000 = 15,000 dollars. The new house is 15,000 dollars. So, the total
-value of the house is 15,000 + 80,000 + 15,000 = 11,000 dollars. The
-total value of the house is 11,000 dollars. The total value of the house
-is 11,000 + 15,000 = 11,000 + 15,000 = 11,000 + 15,000 =</td>
+<td>I: 101.06 ms, T: 0.19 ms</td>
+<td>https://github.com/b4rtaz/distributed-llama/discussions/9</td>
+<td></td>
+<td>(30 vCPU, 15 core, 59 GB memory) europe-west1, AMD Genoa</td>
+<td></td>
+<td></td>
 </tr>
-</tbody>
-</table>
-<h3 id="boolq">boolq</h3>
-<p>而 boolq 相对是一个简单的数据集，输出只需要有 yes or no 即可。</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;conversations&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;user&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Passage: Persian (/ˈpɜːrʒən, -ʃən/), also known by its endonym Farsi (فارسی fārsi (fɒːɾˈsiː) ( listen)), is one of the Western Iranian languages within the Indo-Iranian branch of the Indo-European language family. It is primarily spoken in Iran, Afghanistan (officially known as Dari since 1958), and Tajikistan (officially known as Tajiki since the Soviet era), and some other regions which historically were Persianate societies and considered part of Greater Iran. It is written in the Persian alphabet, a modified variant of the Arabic script, which itself evolved from the Aramaic alphabet.\nQuestion: do iran and afghanistan speak the same language\nAnswer:&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;assistant&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Yes&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">]</span></span><br></pre></td></tr></table></figure>
-<p>在该指标上结果就是相对正常的，符合 Meta 论文的结果。</p>
-<ul>
-<li>裁剪前：～84%</li>
-<li>裁剪后：～68%</li>
-<li>裁剪+训练后：～85%</li>
-</ul>
-<p>所以，模型深度或许是加强模型推理能力的关键因素之一？</p>
-]]></content>
-      <categories>
-        <category>PaperReading</category>
-      </categories>
-      <tags>
-        <tag>NLP</tag>
-        <tag>LLM</tag>
-        <tag>knowledge distillation</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>LLM的推理加速</title>
-    <url>/hexo_blog/2024/06/15/LLM%E7%9A%84%E6%8E%A8%E7%90%86%E5%8A%A0%E9%80%9F/</url>
-    <content><![CDATA[<p>对于两种场景下介绍 LLM 的 推理加速方法</p>
-<p>长输入，短输出</p>
-<p>长输出但输出 token 有限</p>
-<span id="more"></span>
-<p>大语言模型在进行推理时是非常耗时的，因为需要对每个 token
-进行计算。对于推理加速这个话题，在工程上可以从很多方面考虑：</p>
-<ul>
-<li>模型进行量化/剪枝，降低模型浮点计算成本</li>
-<li>Continuous Batch，批量计算 Next Token</li>
-<li>KV Cache：LLM 推理的是 Token by Token
-生成，每次生成的时候会计算所有输入 Token 的
-Attention，故可以缓存已计算的 Attention，减少计算量</li>
-<li>对显存进行管理，如 <a
-href="https://wnma3mz.github.io/hexo_blog/2023/11/24/vLLM%E7%9A%84PageAttention/">PageAttention</a></li>
-<li>KV Cache 量化</li>
-<li>...</li>
-</ul>
-<p>而其他在工程上的优化，往往是针对特定场景的，下面介绍两种场景下的优化方法。</p>
-<h2 id="prefix-cache长输入短输出">Prefix Cache：长输入，短输出</h2>
-<p>注：优化显存并不会减少计算量，只是减少了显存占用，从而提升了吞吐。</p>
-<h3 id="kv-cache-显存计算公式">KV cache 显存计算公式</h3>
-<p><span class="math inline">\(4\times b\times l\times num\_heads\times
-embed\_size\_per\_head \times (s+n)\)</span></p>
-<p>参数说明</p>
-<ul>
-<li>b: 句子条数</li>
-<li>l：层数</li>
-<li>num_heads：隐层大小 (num_key_value_heads)</li>
-<li>embed_size_per_head：每个头的大小 (hidden_size /
-num_attention_heads)</li>
-<li>s：输入长度</li>
-<li>n：输出长度</li>
-<li>4：k cache+v cache，均为 float16，所以是(1+1)*2</li>
-</ul>
-<p>如果只有1条句子，输入+输出 token 长度由 512 -&gt; 1024，则会增加
-<span class="math inline">\(4\times 512 \times l\times h\)</span>：</p>
-<p>Llama3-8B：$4 / 1024 / 1024 = 64 $ M</p>
-<p>Llama3-70B：<span class="math inline">\(4\times 512\times 1024\times
-80 / 1024 / 1024 = 160\)</span> M</p>
-<p>简单来说，每增加一个 token</p>
-<p>8B 就会增加 0.125 M 的显存</p>
-<p>70B 就会增加 0.3125 M 的显存</p>
-<p>这个显存占有直到整个句子生成完成之后才会释放，所以对于长句子，KV
-Cache 的显存占用是非常大的。</p>
-<p>显存决定了整体的吞吐量。对于 70B，假设有 1k 个请求，每个请求的总
-token 为 1k，那么显存占用为 0.3125M * 1000 * 1000 = 0.3G。</p>
-<h3 id="方法">方法</h3>
-<p>在 LLM
-推理时，往往是用对话的方式进行推理，而每次对话的开头可能是相同（System），这一部分的
-Attention 是不变的，所以可以缓存这部分 Attention。</p>
-<p>加速的原因：</p>
-<ul>
-<li>prefix 的 kv cache 不需要计算</li>
-<li>prefix 的 kv cache 复用节约了显存，提升吞吐（主要收益）</li>
-</ul>
-<p>所以适用场景的一个明显特点就是必须要有长的 system prefix，而且这个
-prefix
-是不变的。另外，如果模型输出过长的话，这个优化的收益就会变小。因为它并不会加速在生成过程中的计算，只是减少了显存占用。</p>
-<h3 id="收益">收益</h3>
-<p>https://github.com/vllm-project/vllm/issues/227</p>
-<p>For each request, the prefix length is 200, the input length is 30,
-and the output length is 50.</p>
-<table>
-<colgroup>
-<col style="width: 12%" />
-<col style="width: 15%" />
-<col style="width: 20%" />
-<col style="width: 29%" />
-<col style="width: 20%" />
-</colgroup>
-<thead>
 <tr>
-<th>Load (QPS)</th>
-<th>Method</th>
-<th>Requests/s</th>
-<th>Average Latency per Req</th>
-<th>First Token Time</th>
+<td>c3d-highcpu-30 *4</td>
+<td>Llama 2 7B（Q4）</td>
+<td>53.69 ms</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
-</thead>
-<tbody>
 <tr>
-<td>10 QPS</td>
-<td>Prefix Cache</td>
-<td>9.83 requests/s</td>
-<td>1.97 s</td>
-<td>0.29 s</td>
+<td>I: 40.25 ms, T: 12.81 ms</td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>10 QPS</td>
-<td>Base</td>
-<td>9.80 requests/s</td>
-<td>2.87 s</td>
-<td>0.45 s</td>
+<td>c3d-highcpu-30</td>
+<td>Llama 2 70B（Q4）</td>
+<td>909.69 ms</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>15 QPS</td>
-<td>Prefix Cache</td>
-<td>14.30 requests/s</td>
-<td>2.98 s</td>
-<td>0.39 s</td>
+<td>I: 907.25 ms, T: 1.75 ms</td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>15 QPS</td>
-<td>Base</td>
-<td>13.24 requests/s</td>
-<td>8.65 s</td>
-<td>1.02 s</td>
+<td>c3d-highcpu-30 *4</td>
+<td>Llama 2 70B（Q4）</td>
+<td>293.06 ms</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>25 QPS</td>
-<td>Prefix Cache</td>
-<td>19.81 requests/s</td>
-<td>6.46 s</td>
-<td>0.84 s</td>
+<td>I: 264.00 ms, T: 28.50 ms</td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>25 QPS</td>
-<td>Base</td>
-<td>14.08 requests/s</td>
-<td>13.67 s</td>
-<td>4.74 s</td>
-</tr>
-</tbody>
-</table>
-<h2 id="并行解码长输出但输出-token-有限">并行解码：长输出但输出 token
-有限</h2>
-<p>LLM 的推理是 Token by Token 生成的，如果有一种方式可以每次生成多个
-token，那么就可以提升推理速度。</p>
-<h3 id="方法-1">方法</h3>
-<p>以下图为例，对 LLM 的模型结构进行一定的修改，使每次可以同时生成 3 个
-token。</p>
-<figure>
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/llm_speedup/251e873bb1322b4d31fee19f0c81c424.png"
-alt="并行解码" />
-<figcaption aria-hidden="true">并行解码</figcaption>
-</figure>
-<p>并行解码的方式不一定要对结构进行修改，这里只是一种实现方式。</p>
-<p>具体实现方式： -
-训练：修改结构后对模型的新增部分进行常规的训练，其他部分冻结，减少训练成本
-- 推理：模型生成的时候会把连续 3 个 token 的 top-K
-的可能均生成，交由原模型进行打分，然后选择最优的一个。</p>
-<h3 id="问题">问题</h3>
-<p>并行生成的 Token 的可靠性是取决于具体场景的。如果生成的 Token
-可能性只有 20 个，那么并行生成的 Token
-的可靠性就会相对较高，这种对于模型而言是更容易学习的。但如果生成的 Token
-可能有 10000 个，那么并行生成的 Token 的可靠性就会相对较低。</p>
-<p>换而言之，Token
-可能性越少越容易猜中，越多越难猜中。所以，这种方法适用于输出 token
-可能性有限的场景。</p>
-<p>并且，如果输出 token 数量较短，那么相较于 token by token
-的生成，这种方法的优势就会变小。</p>
+<td>M1</td>
+<td>Llama 7B（Q4）</td>
+<td>14.19 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/discussions/4167</td>
+<td>2023.11.22</td>
+<td>取最慢的速度</td>
+</tr>
+<tr>
+<td>M1 Pro</td>
+<td>Llama 7B（Q4）</td>
+<td>35.52 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>M1 Max</td>
+<td>Llama 7B（Q4）</td>
+<td>54.61 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>M1 Ultra</td>
+<td>Llama 7B（Q4）</td>
+<td>74.93 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>M2</td>
+<td>Llama 7B（Q4）</td>
+<td>21.7 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>M2 Pro</td>
+<td>Llama 7B（Q4）</td>
+<td>37.87 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>M2 Max</td>
+<td>Llama 7B（Q4）</td>
+<td>60.99 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>M2 Ultra</td>
+<td>Llama 7B（Q4）</td>
+<td>65.95 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>M3</td>
+<td>Llama 7B（Q4）</td>
+<td>21.34 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>M3 Pro</td>
+<td>Llama 7B（Q4）</td>
+<td>30.65 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>M3 Max</td>
+<td>Llama 7B（Q4）</td>
+<td>56.58 t/s</td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>AMD EPYC 7443P</td>
+<td>Llama 7B（Q4）</td>
+<td>11.18 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1465138574</td>
+<td>2023.3.12</td>
+<td></td>
+</tr>
+<tr>
+<td>Ryzen 7 3700X</td>
+<td>Llama 7B（Q4）</td>
+<td>8.51 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1465313724</td>
+<td>2023.3.13</td>
+<td></td>
+</tr>
+<tr>
+<td>13900k</td>
+<td>Llama 7B（Q4）</td>
+<td>14.02 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1467067155</td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td>2x Intel Xeon Gold 5120 @ 2.20GHz</td>
+<td>Llama 7B（Q4）</td>
+<td>8.68 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1471171246</td>
+<td>2023.3.16</td>
+<td></td>
+</tr>
+<tr>
+<td>E5-2680v4</td>
+<td>Llama 7B（Q4）</td>
+<td>8.87 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1517704976</td>
+<td>2023.4.21</td>
+<td></td>
+</tr>
+<tr>
+<td>i5 6500</td>
+<td>Llama 7B（Q4）</td>
+<td>13.82 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1550410400</td>
+<td>2023.5.17</td>
+<td></td>
+</tr>
+<tr>
+<td>Hetzner Cloud Arm64 Ampere, 16 VCPU</td>
+<td>Llama 7B（Q4）</td>
+<td>11.76 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1575736794</td>
+<td>2023.6.5</td>
+<td></td>
+</tr>
+<tr>
+<td>13900k</td>
+<td>Llama 7B（Q4）</td>
+<td>12.65 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1675971336</td>
+<td>2023.8.12</td>
+<td></td>
+</tr>
+<tr>
+<td>Snapdragon 870 / 8GB of ram</td>
+<td>zephyr-7b (Q4)</td>
+<td>4.7 t/s</td>
+<td>https://github.com/ggerganov/llama.cpp/issues/34#issuecomment-1825489115</td>
+<td>2023.11.24</td>
+<td></td>
+</tr>
+<tr>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+</tr>
+</tbody>
+</table>
+<p>I ：推理每个 token花费的时间</p>
+<p>T：通信时间</p>
 ]]></content>
       <categories>
         <category>Note</category>
       </categories>
       <tags>
         <tag>NLP</tag>
-        <tag>Attention</tag>
+        <tag>LLM</tag>
+        <tag>Raspberry</tag>
       </tags>
   </entry>
   <entry>
-    <title>MoE中负载均衡Loss实现</title>
-    <url>/hexo_blog/2024/06/15/MoE%E4%B8%AD%E8%B4%9F%E8%BD%BD%E5%9D%87%E8%A1%A1Loss%E5%AE%9E%E7%8E%B0%E5%AF%B9%E6%AF%94/</url>
-    <content><![CDATA[<p>对比 MoE 中负载均衡 Loss 的实现方式，主要有跨层和非跨层两种实现</p>
+    <title>LLM 深度的”稀疏性“</title>
+    <url>/hexo_blog/2024/03/28/LLM%E6%B7%B1%E5%BA%A6%E7%9A%84%E7%A8%80%E7%96%8F%E6%80%A7/</url>
+    <content><![CDATA[<p>LLM 在宽度上的稀疏性可以通过 MoE
+这种架构来优化。而深度上也具备稀疏性，故而可以进行裁剪。</p>
 <span id="more"></span>
-<h2 id="moe-概述">MoE 概述</h2>
-<p>MoE（Mixture of
-Experts）是一种模型结构，由多个专家（expert）组成，每个专家负责处理不同的输入数据。在训练过程中，通过一个
-gating network 来决定每个输入数据由哪个专家处理。</p>
-<figure>
-<img
-src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/moe/00_switch_transformer.png"
-alt="MoE" />
-<figcaption aria-hidden="true">MoE</figcaption>
-</figure>
-<p>在 LLM 的 Next Token 训练方式下，每个 token
-会被分配到一个专家处理，所以需要保证每个专家被选中的次数相等，这样才能保证每个专家都能得到充分的训练。为此，需要引入负载均衡
-Loss。</p>
-<h2 id="负载均衡-loss">负载均衡 Loss</h2>
-<p>在<a
-href="https://arxiv.org/abs/2101.03961">原论文</a>中公式如下：</p>
-<figure>
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/MoE/load_balancing_loss.png"
-alt="load_balancing_loss" />
-<figcaption aria-hidden="true">load_balancing_loss</figcaption>
-</figure>
-<p>但这个公式表达的信息不够全面。在深度神经网络中，往往具备很多层，即每层都具备混合专家。因此，这里其实有两种实现方式：</p>
+<h2 id="背景">背景</h2>
+<p>这篇<a
+href="https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens">博客</a>发现到
+GPT-2 在深度上的输出具有相似性。即，第 20 层的输出接入 LM Head 和第 21
+层的输出接入 LM Head 是相似，甚至一致的。</p>
+<p>故而，模型在深度上可能是稀疏的。所以可以按 block
+的维度进行裁剪。如下图所示，</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/image-2024-3-28_18-23-0.png" /></p>
+<p>找到三篇相似的工作</p>
 <ul>
-<li>跨层实现：对于所有 token，期望所有层选出来的专家次数相等</li>
-<li>非跨层实现：对于所有 token，期望每一层选出来的专家次数相等</li>
-</ul>
-<p>举个例子，假设两层的神经网络，两个专家，四个 token。</p>
-<p>非跨层：对于所有 token，期望每一层选出来的专家次数相等。所以第一层 1
-号专家被选中 2 次，2 号专家被选中 2 次；第二层一样。 跨层：对于所有
-token，期望所有层选出来的专家次数相等。所以可以第一层 1 号专家被选中 4
-次，第二层 2 号专家被选中 4 次。求和，每个专家被选中的次数相等</p>
-<p>换而言之，相当于跨层实现是一种更松散的实现方式，并不要求每层每个专家被选中的次数相等，只要整体均衡即可。</p>
-<p>huggingface
-的实现：https://github.com/huggingface/transformers/blob/main/src/transformers/models/mixtral/modeling_mixtral.py</p>
-<p>megatron 的实现：
-https://github.com/databricks/megablocks/blob/main/megablocks/layers/moe.py</p>
-<p>下面是
-https://gist.github.com/tdrussell/0529afd8d280fbe2c1c582d8f865e909
-实现的两种方式的对比。</p>
-<h3 id="跨层实现">跨层实现</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">load_balancing_loss_func1</span>(<span class="params">gate_logits: torch.Tensor, num_experts: torch.Tensor = <span class="literal">None</span>, top_k=<span class="number">2</span></span>) -&gt; <span class="built_in">float</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(gate_logits, <span class="built_in">tuple</span>):</span><br><span class="line">        compute_device = gate_logits[<span class="number">0</span>].device</span><br><span class="line">        concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) <span class="keyword">for</span> layer_gate <span class="keyword">in</span> gate_logits], dim=<span class="number">0</span>)</span><br><span class="line"> </span><br><span class="line">    routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-<span class="number">1</span>)</span><br><span class="line">    _, selected_experts = torch.topk(routing_weights, top_k, dim=-<span class="number">1</span>) <span class="comment"># [batch_size X sequence_length, top_k]</span></span><br><span class="line">    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) <span class="comment"># [batch_size X sequence_length, top_k, num_experts]</span></span><br><span class="line">    tokens_per_expert = torch.mean(expert_mask.<span class="built_in">float</span>(), dim=<span class="number">0</span>) <span class="comment"># [top_k, num_experts]</span></span><br><span class="line">    <span class="comment"># Compute the average probability of routing to these experts</span></span><br><span class="line">    router_prob_per_expert = torch.mean(routing_weights, dim=<span class="number">0</span>) <span class="comment"># [num_experts]</span></span><br><span class="line">    overall_loss = torch.<span class="built_in">sum</span>(tokens_per_expert * router_prob_per_expert.unsqueeze(<span class="number">0</span>)) <span class="comment"># / top_k</span></span><br><span class="line">    <span class="keyword">return</span> overall_loss * num_experts</span><br></pre></td></tr></table></figure>
-<h3 id="非跨层实现">非跨层实现</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">load_balancing_loss_func2</span>(<span class="params">gate_logits: torch.Tensor, num_experts: torch.Tensor = <span class="literal">None</span>, top_k=<span class="number">2</span></span>) -&gt; <span class="built_in">float</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(gate_logits, <span class="built_in">tuple</span>):</span><br><span class="line">        compute_device = gate_logits[<span class="number">0</span>].device</span><br><span class="line">        stacked_gate_logits = torch.stack([layer_gate.to(compute_device) <span class="keyword">for</span> layer_gate <span class="keyword">in</span> gate_logits], dim=<span class="number">0</span>)</span><br><span class="line"> </span><br><span class="line">    routing_weights = torch.nn.functional.softmax(stacked_gate_logits, dim=-<span class="number">1</span>) <span class="comment"># [num_layers, num_tokens, num_experts]</span></span><br><span class="line">    _, selected_experts = torch.topk(routing_weights, top_k, dim=-<span class="number">1</span>) <span class="comment"># [num_layers, num_tokens, top_k]</span></span><br><span class="line">    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) <span class="comment"># [num_layers, num_tokens, top_k, num_experts]</span></span><br><span class="line">    <span class="comment"># For a given token, determine if it was routed to a given expert. Think of this as a collection of top_k-hot vectors.</span></span><br><span class="line">    expert_mask = torch.<span class="built_in">max</span>(expert_mask, dim=-<span class="number">2</span>).values.<span class="built_in">float</span>() <span class="comment"># [num_layers, num_tokens, num_experts]</span></span><br><span class="line">    tokens_per_layer_and_expert = torch.mean(expert_mask, dim=-<span class="number">2</span>) <span class="comment"># [num_layers, num_experts]</span></span><br><span class="line">    router_prob_per_layer_and_expert = torch.mean(routing_weights, dim=-<span class="number">2</span>) <span class="comment"># [num_layers, num_experts]</span></span><br><span class="line">    <span class="keyword">return</span> torch.mean(tokens_per_layer_and_expert * router_prob_per_layer_and_expert) * num_experts**<span class="number">2</span></span><br></pre></td></tr></table></figure>
-<h3 id="对比">对比</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    gate_logits1 = torch.tensor([<span class="number">5</span>, <span class="number">1</span>, <span class="number">0</span>, <span class="number">0</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits2 = torch.tensor([<span class="number">0</span>, <span class="number">5</span>, <span class="number">1</span>, <span class="number">0</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits3 = torch.tensor([<span class="number">0</span>, <span class="number">0</span>, <span class="number">5</span>, <span class="number">1</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits4 = torch.tensor([<span class="number">1</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">5</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits = (gate_logits1, gate_logits2, gate_logits3, gate_logits4)</span><br><span class="line"> </span><br><span class="line">    <span class="built_in">print</span>(load_balancing_loss_func1(gate_logits, num_experts=<span class="number">4</span>))  <span class="comment"># 2.0</span></span><br><span class="line">    <span class="built_in">print</span>(load_balancing_loss_func2(gate_logits, num_experts=<span class="number">4</span>))  <span class="comment"># 3.9478</span></span><br></pre></td></tr></table></figure>
-<p>简单来看，非跨层实现能够对每一层的专家进行更强的约束，预期会实现更好的负载均衡。所以，下面引入模型和数据，来对比这种负载均衡
-Loss 两种写法。</p>
-<h2 id="loss-实现">Loss 实现</h2>
-<p>在 PyTorch 实现这种网络层中间的 Loss，可以有两种方法：</p>
-<h3 id="简单直接">简单直接</h3>
-<p>直接在网络结构中引入一个函数来计算 loss
-，最后把这个值返回模型输出，依赖最外面的<code>loss.backward()</code>
-进行梯度更新。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">criterion</span>(<span class="params">self, x</span>):</span><br><span class="line">    <span class="keyword">return</span> torch.mean(x**<span class="number">2</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h3 id="torch.autograd.function">torch.autograd.Function</h3>
-<p>一种更高级可控的方式是使用
-<code>torch.autograd.Function</code>，这种方式可以更好的控制梯度的传递，可以在这个函数中对梯度进行缩放。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MoEAuxLossAutoScaler</span>(torch.autograd.Function):</span><br><span class="line">    main_loss_backward_scale: torch.Tensor = torch.tensor(<span class="number">1.0</span>)</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, output: torch.Tensor, aux_loss: torch.Tensor</span>):</span><br><span class="line">        ctx.save_for_backward(aux_loss)</span><br><span class="line">        <span class="keyword">return</span> output</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output: torch.Tensor</span>):</span><br><span class="line">        (aux_loss,) = ctx.saved_tensors</span><br><span class="line">        aux_loss_backward_scale = MoEAuxLossAutoScaler.main_loss_backward_scale</span><br><span class="line">        scaled_aux_loss_grad = torch.ones_like(aux_loss) * aux_loss_backward_scale</span><br><span class="line">        <span class="keyword">return</span> grad_output, scaled_aux_loss_grad</span><br></pre></td></tr></table></figure>
-<h3 id="完整例子">完整例子</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> torch.nn <span class="keyword">as</span> nn</span><br><span class="line"><span class="keyword">import</span> torch.optim <span class="keyword">as</span> optim</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">seed_everything</span>(<span class="params">seed</span>):</span><br><span class="line">    random.seed = seed</span><br><span class="line">    np.random.seed(seed)</span><br><span class="line">    torch.manual_seed(seed)</span><br><span class="line">    torch.cuda.manual_seed_all(seed)</span><br><span class="line">    torch.backends.cudnn.deterministic = <span class="literal">True</span></span><br><span class="line">    torch.backends.cudnn.benchmark = <span class="literal">False</span></span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line">seed_everything(<span class="number">42</span>)</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MoEAuxLossAutoScaler</span>(torch.autograd.Function):</span><br><span class="line">    main_loss_backward_scale: torch.Tensor = torch.tensor(<span class="number">1.0</span>)</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, output: torch.Tensor, aux_loss: torch.Tensor</span>):</span><br><span class="line">        ctx.save_for_backward(aux_loss)</span><br><span class="line">        <span class="keyword">return</span> output</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output: torch.Tensor</span>):</span><br><span class="line">        (aux_loss,) = ctx.saved_tensors</span><br><span class="line">        aux_loss_backward_scale = MoEAuxLossAutoScaler.main_loss_backward_scale</span><br><span class="line">        scaled_aux_loss_grad = torch.ones_like(aux_loss) * aux_loss_backward_scale</span><br><span class="line">        <span class="keyword">return</span> grad_output, scaled_aux_loss_grad</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="comment"># 创建自定义模型</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyModel</span>(nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, num_layers=<span class="number">1</span></span>):</span><br><span class="line">        <span class="built_in">super</span>(MyModel, <span class="variable language_">self</span>).__init__()</span><br><span class="line">        <span class="variable language_">self</span>.num_layers = num_layers</span><br><span class="line">        <span class="variable language_">self</span>.same_layer = nn.ModuleList([nn.Linear(<span class="number">20</span>, <span class="number">20</span>, bias=<span class="literal">False</span>) <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(num_layers)])</span><br><span class="line">        <span class="variable language_">self</span>.lm_head = nn.Linear(<span class="number">20</span>, <span class="number">1</span>, bias=<span class="literal">False</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">criterion</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> torch.mean(x**<span class="number">2</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x, is_complex=<span class="literal">False</span></span>):</span><br><span class="line">        lbl_loss = <span class="number">0.0</span></span><br><span class="line">        out1 = x</span><br><span class="line">        <span class="keyword">for</span> i, layer <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="variable language_">self</span>.same_layer):</span><br><span class="line">            out1 = layer(out1)</span><br><span class="line"> </span><br><span class="line">            lbl_loss_layer = <span class="variable language_">self</span>.criterion(out1)</span><br><span class="line">            <span class="comment"># 如果是复杂的方法，需要使用 MoEAuxLossAutoScaler，将梯度缩放到主要的 loss 上</span></span><br><span class="line">            <span class="comment"># 这样不需要返回中间计算的 loss 结果</span></span><br><span class="line">            <span class="keyword">if</span> is_complex:</span><br><span class="line">                out1 = MoEAuxLossAutoScaler.apply(out1, lbl_loss_layer / <span class="variable language_">self</span>.num_layers)</span><br><span class="line">            <span class="comment"># 如果是简单的方法，直接返回中间计算的 loss 结果</span></span><br><span class="line">            <span class="comment"># 但这样需要把 loss 计算的结果返回，以在外部进行梯度更新</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                lbl_loss += (lbl_loss_layer / <span class="variable language_">self</span>.num_layers)</span><br><span class="line">        out3 = <span class="variable language_">self</span>.lm_head(out1)</span><br><span class="line">        <span class="keyword">return</span> out3, lbl_loss</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    device = <span class="string">&quot;cpu&quot;</span></span><br><span class="line">    num_layers = <span class="number">2</span></span><br><span class="line">    model = MyModel(num_layers).to(device)</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 定义优化器</span></span><br><span class="line">    optimizer = optim.SGD(model.parameters(), lr=<span class="number">1e-4</span>)</span><br><span class="line">    criterion = nn.MSELoss()</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 输入随机数据</span></span><br><span class="line">    input_ = torch.randn(<span class="number">10</span>, <span class="number">20</span>).to(device)</span><br><span class="line">    real_out = torch.randn(<span class="number">10</span>, <span class="number">1</span>).to(device)</span><br><span class="line"> </span><br><span class="line">    is_complex = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">for</span> <span class="built_in">iter</span> <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">100</span>):</span><br><span class="line">        out, lbl_loss = model(input_, is_complex=is_complex)</span><br><span class="line"> </span><br><span class="line">        <span class="comment"># 如果是简单方法是有值的</span></span><br><span class="line">        <span class="comment"># 如果是复杂方法 lbl_loss 是 0</span></span><br><span class="line">        loss = lbl_loss + criterion(out, real_out)</span><br><span class="line"> </span><br><span class="line">        optimizer.zero_grad()</span><br><span class="line">        loss.backward()</span><br><span class="line">        optimizer.step()</span><br><span class="line"> </span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;is_complex: <span class="subst">&#123;is_complex&#125;</span>&quot;</span>, model.state_dict()[<span class="string">f&quot;same_layer.0.weight&quot;</span>][<span class="number">0</span>])  </span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<li>上交：<a href="https://arxiv.org/html/2402.11187v1">LaCo: Large
+Language Model Pruning via Layer Collapse</a></li>
+<li>Meta：<a href="https://arxiv.org/html/2403.17887">The Unreasonable
+Ineffectiveness of the Deeper Layers</a></li>
+<li>百川：<a href="https://arxiv.org/html/2403.03853v2">ShortGPT: Layers
+in Large Language Models are More Redundant Than You Expect</a></li>
+</ul>
+<p>越靠近模型输出的那些层相似度较高，可以通过删除一些“冗余”层来减小模型参数</p>
+<h2 id="方法的共通性">方法的共通性</h2>
+<ul>
+<li>构建一个“中性”数据集！！！</li>
+<li>观察不同层的输入/输出</li>
+<li>用某种度量距离的方法测量不同层是否相似
+<ul>
+<li>百川：Block Influence</li>
+<li>Meta：角距离</li>
+</ul></li>
+</ul>
+<h2 id="meta-论文结果">Meta 论文结果</h2>
+<h3 id="不同模型具有相似的表现">不同模型具有相似的表现</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/x2.png" /></p>
+<p>注：</p>
+<ul>
+<li>实线（healing）：扔掉一些层之后进行了训练</li>
+<li>虚线（no healing）： → 扔掉这些层后不训练</li>
+<li>实线和虚线扔掉的是相同的层</li>
+<li>横坐标是扔掉的比例</li>
+</ul>
+<p>Q：MMLU/BoolQ 指标具备足够的可信度？</p>
+<p>A：个人感觉没有，但有后续有其他的一些实验支撑</p>
+<p>Q：Qwen 为啥表现不大一样？</p>
+<p>A：论文其他部分进行一定的解释，但与本文的重点相关性不强，所以不展开介绍</p>
+<h3 id="暴力裁剪-vs-提出的方法">暴力裁剪 vs 提出的方法</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/x5.png" /></p>
+<p>有一种直接粗暴的方法就是直接扔掉最后若干层，这样就不需要数据集啥的来选层。</p>
+<p>奇怪的现象就是直接扔掉最后若干层，指标会出现先降低再提升（不训练的情况）。</p>
+<h2 id="百川论文的结论">百川论文的结论</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/sparseinLLM/baichuan.png" /></p>
+<p><strong>第一层和最后一层很重要！！！</strong></p>
+<h2 id="剪枝比率">剪枝比率</h2>
+<p>利用宽度稀疏性，一种优化架构 MoE
+可以做到降低一半的参数且保持性能不变。</p>
+<p>而在深度稀疏上，这一点是比较难做到的。</p>
+<table>
+<thead>
+<tr>
+<th>模型尺寸</th>
+<th>层数</th>
+<th>参数量</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>70B</td>
+<td>80</td>
+<td>69,434,351,616</td>
+</tr>
+<tr>
+<td>70B</td>
+<td>40</td>
+<td>35,208,175,616</td>
+</tr>
+<tr>
+<td>70B</td>
+<td>45</td>
+<td>39,486,447,616</td>
+</tr>
+<tr>
+<td>34B</td>
+<td>48</td>
+<td>34,201,673,728</td>
+</tr>
+<tr>
+<td>34B</td>
+<td>24</td>
+<td>17,591,836,672</td>
+</tr>
+</tbody>
+</table>
+<p>简单来说，对于 70B 模型要剪枝到 34B，需要砍掉一半的层；对于 34B
+模型要剪枝到 13B，要砍掉大于一半的层。</p>
+<p>通过 Meta 论文的实验结果显示，当裁剪超过
+40%，就会有较大的风险不能恢复模型性能。</p>
+<h2 id="实验">实验</h2>
+<p>在 boolq/gsm8k
+数据集上简单了进行若干实验，这两个数据集均具备训练集和评测集。所以，对
+13B 模型进行暴力裁剪后（～40%
+参数）在训练集上进行训练，在评测集上进行评测。</p>
+<ul>
+<li>裁剪方法：保留最后一层，从最后一层往前数，直至裁剪到 40% 为止。</li>
+<li>完整训练集：对应评测的训练集+通用的QA/对话数据集</li>
+</ul>
+<h3 id="gsm8k">gsm8k</h3>
+<p>是一个考验数学/逻辑推理能力的评测结果</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;conversations&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;user&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;assistant&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Natalia sold 48 clips in April.\nThen she sold half as many in May, which is 48 / 2 = 24.\nSo altogether she sold 48 + 24 = 72 clips.\nThe answer is 72.&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">]</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
+<p>在该评测上指标直接崩掉，如原来可能是 &gt;70，现在只有 2%</p>
+<table>
+<colgroup>
+<col style="width: 47%" />
+<col style="width: 52%" />
+</colgroup>
+<thead>
+<tr>
+<th><strong>prompt</strong></th>
+<th><strong>pred</strong></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>Janet’s ducks lay 16 eggs per day. She eats three for breakfast
+every morning and bakes muffins for her friends every day with four. She
+sells the remainder at the farmers' market daily for $2 per fresh duck
+egg. How much in dollars does she make every day at the farmers' market?
+Let's think step by step.</td>
+<td>Each egg is 16 eggs/day for 16 eggs/day for 1600000000 / 16 =
+128000000. So, the total number of eggs is 128000000. The total number
+of eggs is 160000000 - 128000000 = 128000000.The answer is
+128000000.</td>
+</tr>
+<tr>
+<td>A robe takes 2 bolts of blue fiber and half that much white fiber.
+How many bolts in total does it take? Let's think step by step</td>
+<td>The rose is 2 bolts of blue fibers. Half that amount is 2 / 2 = 1.
+So the rose is 2 - 1 = 2 flobs. The answer is 2.</td>
+</tr>
+<tr>
+<td>Josh decides to try flipping a house. He buys a house for $80,000
+and then puts in $50,000 in repairs. This increased the value of the
+house by 150%. How much profit did he make? Let's think step by
+step.</td>
+<td>The house was originally 80,000 dollars. The new house is 80,000 +
+50,000 = 15,000 dollars. The new house is 15,000 dollars. So, the total
+value of the house is 15,000 + 80,000 + 15,000 = 11,000 dollars. The
+total value of the house is 11,000 dollars. The total value of the house
+is 11,000 + 15,000 = 11,000 + 15,000 = 11,000 + 15,000 =</td>
+</tr>
+</tbody>
+</table>
+<h3 id="boolq">boolq</h3>
+<p>而 boolq 相对是一个简单的数据集，输出只需要有 yes or no 即可。</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;conversations&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;user&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Passage: Persian (/ˈpɜːrʒən, -ʃən/), also known by its endonym Farsi (فارسی fārsi (fɒːɾˈsiː) ( listen)), is one of the Western Iranian languages within the Indo-Iranian branch of the Indo-European language family. It is primarily spoken in Iran, Afghanistan (officially known as Dari since 1958), and Tajikistan (officially known as Tajiki since the Soviet era), and some other regions which historically were Persianate societies and considered part of Greater Iran. It is written in the Persian alphabet, a modified variant of the Arabic script, which itself evolved from the Aramaic alphabet.\nQuestion: do iran and afghanistan speak the same language\nAnswer:&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="punctuation">&#123;</span></span><br><span class="line">        <span class="attr">&quot;role&quot;</span><span class="punctuation">:</span> <span class="string">&quot;assistant&quot;</span><span class="punctuation">,</span></span><br><span class="line">        <span class="attr">&quot;content&quot;</span><span class="punctuation">:</span> <span class="string">&quot;Yes&quot;</span></span><br><span class="line">    <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">]</span></span><br></pre></td></tr></table></figure>
+<p>在该指标上结果就是相对正常的，符合 Meta 论文的结果。</p>
+<ul>
+<li>裁剪前：～84%</li>
+<li>裁剪后：～68%</li>
+<li>裁剪+训练后：～85%</li>
+</ul>
+<p>所以，模型深度或许是加强模型推理能力的关键因素之一？</p>
 ]]></content>
       <categories>
-        <category>Note</category>
+        <category>PaperReading</category>
       </categories>
       <tags>
-        <tag>MoE</tag>
-        <tag>Loss</tag>
+        <tag>NLP</tag>
+        <tag>LLM</tag>
+        <tag>knowledge distillation</tag>
       </tags>
   </entry>
   <entry>
-    <title>SPSA随机优化算法</title>
-    <url>/hexo_blog/2023/08/19/SPSA%E9%9A%8F%E6%9C%BA%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95/</url>
-    <content><![CDATA[<p>SPSA(Simultaneous Perturbation Stochastic
-Approximation)是一种随机优化算法,可用于神经网络的训练。其基于随机梯度来逼近真实梯度。无法直接得到精确梯度,收敛速度较慢,但节省计算资源。</p>
+    <title>LLM的推理相关计算公式</title>
+    <url>/hexo_blog/2024/09/05/LLM%E6%8E%A8%E7%90%86%E7%9B%B8%E5%85%B3%E8%AE%A1%E7%AE%97%E5%85%AC%E5%BC%8F/</url>
+    <content><![CDATA[<p>记录 LLM 在推理上的理论计算公式</p>
 <span id="more"></span>
-<h3 id="spsa算法简介">SPSA算法简介</h3>
-<p>SPSA是一种随机优化算法,可以用于神经网络模型的训练。它的主要思想是同时随机扰动所有参数,基于损失函数在扰动后的变化来估计梯度。</p>
-<p>相比确定性的梯度下降法,SPSA依靠随机性搜索最优解,可以更好地跳出局部最优,具有全局搜索能力。另外,SPSA只需要前向传播就可以更新参数,省去了存储反向传播所需的计算图,计算开销较小。</p>
-<h3 id="pytorch实现spsa训练神经网络">PyTorch实现SPSA训练神经网络</h3>
-<p>要在PyTorch中实现SPSA算法训练神经网络,主要分以下几个步骤:</p>
-<ol type="1">
-<li><p>构建神经网络模型</p>
-<p>可以构建一个多层全连接网络,例如在MNIST数据集上进行图像分类任务:</p>
-<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch.nn <span class="keyword">as</span> nn</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Net</span>(nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="built_in">super</span>(Net, <span class="variable language_">self</span>).__init__()</span><br><span class="line">        <span class="variable language_">self</span>.fc1 = nn.Linear(<span class="number">784</span>, <span class="number">100</span>)</span><br><span class="line">        <span class="variable language_">self</span>.fc2 = nn.Linear(<span class="number">100</span>, <span class="number">10</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line">        x = F.relu(<span class="variable language_">self</span>.fc1(x))</span><br><span class="line">        x = <span class="variable language_">self</span>.fc2(x)</span><br><span class="line">        <span class="keyword">return</span> F.log_softmax(x, dim=<span class="number">1</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure></p></li>
-<li><p>定义损失函数和优化器</p>
-<p>使用交叉熵损失函数和SGD优化器:</p>
-<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line">criterion = nn.CrossEntropyLoss()</span><br><span class="line">optimizer = torch.optim.SGD(net.parameters(), lr=<span class="number">0.1</span>)</span><br></pre></td></tr></table></figure></p></li>
-<li><p>实现SPSA算法</p>
-<p>根据SPSA的公式,可以实现计算随机梯度的函数:</p>
-<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">SPSA_step</span>(<span class="params">net, criterion, x, y, a, c, k</span>):</span><br><span class="line">    delta = <span class="number">2</span> * (torch.randint(<span class="number">0</span>, <span class="number">2</span>, net.parameters().__len__()) - <span class="number">0.5</span>)</span><br><span class="line">    perturbation = c * delta</span><br><span class="line">    loss1 = criterion(net(x + a*perturbation), y)</span><br><span class="line">    loss2 = criterion(net(x - a*perturbation), y)</span><br><span class="line">    gradient = (loss1 - loss2) / (<span class="number">2</span> * a * perturbation)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> param, grad <span class="keyword">in</span> <span class="built_in">zip</span>(net.parameters(), gradient):</span><br><span class="line">        param.data.add_(k * grad * delta)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> net</span><br></pre></td></tr></table></figure></p></li>
-<li><p>训练神经网络</p>
-<p>在训练循环中调用SPSA_step函数计算梯度并更新参数:</p>
-<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">    running_loss = <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> i, data <span class="keyword">in</span> <span class="built_in">enumerate</span>(trainloader, <span class="number">0</span>):</span><br><span class="line">        inputs, labels = data</span><br><span class="line">        </span><br><span class="line"></span><br><span class="line">        net = SPSA_step(net, criterion, inputs, labels, <span class="number">0.01</span>, <span class="number">0.005</span>, <span class="number">0.01</span>)</span><br><span class="line">				optimizer.zero_grad()</span><br><span class="line">        optimizer.step()</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;[%d] loss: %.3f&#x27;</span> % (epoch + <span class="number">1</span>, running_loss / <span class="built_in">len</span>(trainloader)))</span><br><span class="line"></span><br></pre></td></tr></table></figure></p></li>
-</ol>
-<h3 id="前向传播与反向传播的比较">前向传播与反向传播的比较</h3>
-<p>使用前向传播计算梯度时,需要使用两点式差分来估计梯度:</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">gradient = (loss1 - loss2) / (<span class="number">2</span> * a * perturbation)</span><br></pre></td></tr></table></figure>
-<p>其中loss1和loss2分别对应参数做微小正负扰动时的损失。这只能得到梯度的近似值,收敛速度较慢。</p>
-<p>而反向传播可以通过动态计算图高效地得到每个参数的精确梯度,使得网络可以更快收敛。但需要额外的存储空间保存计算图,计算复杂度也更高。</p>
-<p>所以在训练神经网络时,通常选择反向传播来获取精确梯度信息,以保证收敛速度。SPSA更适用于一些计算资源受限但需要求解最优参数的场景。</p>
-<h3 id="q-a">Q &amp; A</h3>
-<p><strong>问题1:SPSA算法与随机梯度下降(SGD)的区别是什么?</strong></p>
-<p>SPSA和SGD都依赖于随机性来达到优化目的,主要区别在于:</p>
+<h3 id="参数量">参数量</h3>
+<p>embedding + lm head + transformer block <span
+class="math inline">\(\times\)</span> N</p>
 <ul>
-<li>SGD每次只随机扰动一个参数,SPSA同时扰动所有参数</li>
-<li>SPSA使用有限差分法估计梯度,SGD使用反向传播计算精确梯度</li>
-<li>SPSA可能更能避免局部最优,SGD收敛速度更快</li>
-</ul>
-<p>使用前向传播和反向传播的主要不同是在计算梯度的方式上。</p>
-<p>使用反向传播时，我们首先通过前向传播计算网络的输出，然后将输出与实际标签计算损失函数，然后通过反向传播计算每个参数对损失函数的梯度。这使得我们能够有效地更新模型参数，并最小化损失函数。</p>
-<p>而如果只使用前向传播进行神经网络的更新，则无法直接计算每个参数对损失函数的梯度。相反，我们可以考虑使用“两点式差分”来近似计算梯度。具体地，我们采用以下公式：</p>
-<p><span class="math display">\[\nabla f(x) \approx \frac{f(x+\Delta x)
-- f(x-\Delta x)}{2\Delta x}\]</span></p>
-<p>其中，<span class="math inline">\(\Delta
-x\)</span>代表一个微小的扰动，<span
-class="math inline">\(x\)</span>是输入数据，<span
-class="math inline">\(f(x)\)</span>为损失函数。我们根据这个公式对每个参数进行微小的扰动，然后计算对应的损失函数。在此基础上，我们可以使用公式计算梯度并更新参数。</p>
-<p>使用前向传播进行更新相比使用反向传播的优势在于，它减少了内存占用和计算时间，并且不需要存储计算图。但是，由于它只是使用一阶近似来计算梯度，因此可能会导致更新不如反向传播精确。</p>
-<p><strong>问题2:SPSA算法适用于哪些场景?</strong></p>
-<p>SPSA算法适用于以下场景:</p>
+<li>Embedding / LM Head 的参数量：vocab_size <span
+class="math inline">\(\times\)</span> hidden_size</li>
+<li>transformer block 的参数量
 <ul>
-<li>目标函数无法明确表达,无法求导</li>
-<li>参数空间高维,难以精确计算梯度</li>
-<li>需要避免陷入局部最优</li>
-<li>计算资源受限,无法存储大规模计算图</li>
-</ul>
-<p><strong>问题3:SPSA算法的两大超参数a和c表示什么?</strong></p>
+<li>self-attention：qkvo 四个线性层，需要分别考虑两种 Attention
 <ul>
-<li>a是步长(perturbation size),表示参数扰动大小</li>
-<li>c是抖动幅度(perturbation distribution),控制扰动值波动范围</li>
-</ul>
-<p>a可类比SGD中的学习率,c越大随机性越强。两者需要均衡调节,既保证足够探索又不失稳定性。</p>
-<p><strong>问题4:SPSA算法如何保证收敛性?</strong></p>
-<p>SPSA可以通过以下手段保证收敛:</p>
-<ol type="1">
-<li>同时扰动参数：在每个迭代中，SPSA使用两个随机向量对参数进行扰动，这样可以避免局部极小值点，并且使得算法更容易跳出局部最小值。</li>
-<li>适应性调整步长：SPSA具有自适应学习率机制，即根据当前估计的梯度大小和先前的步长大小来更新步长。这确保了算法收敛时步长逐渐减小。</li>
-<li>随机性：SPSA在每个迭代中都使用随机扰动来估计梯度，这有助于避免陷入局部最小值并提高全局搜索的能力。</li>
-<li>理论保证：SPSA有理论上保证的收敛性，这意味着只要满足一些条件（例如随机扰动独立同分布），就可以保证算法最终会收敛到正确的解。</li>
-</ol>
-<p><strong>问题5:相较于反向传播。为什么SPSA没有被广泛应用？</strong></p>
-<ol type="1">
-<li>相对较少的理论支持：与反向传播等算法相比，SPSA算法的理论分析较少，因此我们不能完全了解其性能或收敛速度。</li>
-<li>需要大量的迭代次数：SPSA算法需要使用更多的迭代次数才能获得较好的结果，这可能会使其在实际使用中变得过于耗时。</li>
-<li>参数设置困难：SPSA算法本身有很多参数需要调节，例如步长、抖动幅度等，如果这些参数设置不当，则可能导致算法无法收敛或收敛非常缓慢。</li>
-</ol>
-]]></content>
-      <categories>
-        <category>DeepLearning</category>
-      </categories>
-      <tags>
-        <tag>Algorithm</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Short Paper Reading</title>
-    <url>/hexo_blog/2022/12/31/ShortPaperReading/</url>
-    <content><![CDATA[<p>读研期间，短时 paper reading</p>
-<span id="more"></span>
-<h1 id="what-does-the-gradient-represent">What does the gradient
-represent?</h1>
-<p>2020.10.16</p>
-<h2 id="motivation">Motivation</h2>
-<p>The communication in distributed gradient descent. Dense &lt;&lt;
-Sparse</p>
-<h2 id="method">Method</h2>
-<p>Gradient Dropping: removing the R% smallest gradients by absolute
-value</p>
-<h2 id="result">Result</h2>
-<p>Reduce 50x communication size，speed up 22%</p>
-<h2 id="inspirer">Inspirer</h2>
-<p>Gradient value could show the importance of client/model</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image2.png" /></p>
-<p>作者发现分布式机器学习中，梯度大多为稀疏的（大部分接近0），即意味着大部分客户端对梯度更新贡献很小，故可以通过剔除这部分梯度来提升通信效率。</p>
-<p>对R%小的梯度值不进行通信，换而言之，只有满足条件的梯度才会进行通信。</p>
-<p>实验结果证明，该方法能够在精度不变的前提下，减少50倍的通信规模，并且提升了22%的速度</p>
-<p>这篇文章可以间接为受控共享学习中，之前的实验结果做一个解释，只接受梯度值较大的客户端，即表明该客户端的贡献较大。</p>
-<p>问题：分布式机器学习的任务，数据可能都是来源同一个域内的，甚至就是相同的数据；从该文中的图来看，不能证明之前实验收敛速度加快的原因</p>
-<p>Alham Fikri Aji and Kenneth Heafield. Sparse communication for
-distributed gradient descent. In Empirical Methods in Natural Language
-Processing (EMNLP), 2017.</p>
-<h1 id="knowledge-distillation-with-svd">Knowledge Distillation with
-SVD</h1>
-<p>2020.10.23</p>
-<p>Knowledge Distillation (KD) - Fully Connected Layers-&gt;L2 loss</p>
-<p>KD-FSP - Feature Map-&gt;L2 loss</p>
-<p>KD-SVD - Feature Map-&gt;SVD</p>
-<p>KD-EID - KD-SVD(Adaptively)</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image3.png" /></p>
-<p>基于SVD对模型进行压缩的新改进。</p>
-<p>首先对知识蒸馏进行简单的回顾，知识蒸馏是利用教师模型与学生模型在最后输出的全连接层上计算L2损失，以此来达到知识迁移、模型压缩、模型防御等目的。基于KD的改进之一，就是对教师与学生模型中的网络中间层的Feature
-Map进行损失计算。作者基于FSP此类思想，提出对Feature
-Map进行SVD，这样可以有效提取教师模型中的知识，更准确地传递给学生信息。</p>
-<p>新发表的文章，则是对这种方法进行了改进，无需人工干预参数设计。实验证明该方法也是有效的（在TinyImageNet对比原始的学生模型提高了2.89%）。文章大部分内容都还是ECCV那篇文章的内容，增加的自适应部分是判断教师与学生模型的Feature
-Map大小差异，来决定使用SVD还是特征分解。</p>
-<p>Lee S, Song B C. Knowledge Transfer via Decomposing Essential
-Information in Convolutional Neural Networks[J]. IEEE transactions on
-neural networks and learning systems.</p>
-<h1 id="knowledge-distillation-in-federated-learning">Knowledge
-Distillation in Federated Learning</h1>
-<p>2020.10.30</p>
-<h2 id="contributions">Contributions</h2>
-<ul>
-<li><p>distillation framework for robust federated model fusion</p></li>
-<li><p>CV/NLP datasets</p></li>
-<li><p>heterogeneous models and/or data</p></li>
-</ul>
-<h2 id="understanding">Understanding</h2>
+<li>multi-head attention（MHA）：4 <span
+class="math inline">\(\times\)</span> hidden_size <span
+class="math inline">\(^2\)</span></li>
+<li>grouped query attention（GQA）：
 <ul>
-<li><p>Generalization bound.</p></li>
-<li><p>Source, diversity and size of the distillation dataset.</p></li>
-<li><p>Distillation steps.</p></li>
+<li><p>q 和 o，2 $$ hidden_size <span
+class="math inline">\(^2\)</span></p></li>
+<li><p>k 和 v，2 $$ hidden_size <span class="math inline">\(^2\)</span>
+/ num_attention_heads $$ num_key_value_heads</p></li>
+</ul></li>
+</ul></li>
+<li>FFN：三个线性层，3 <span class="math inline">\(\times\)</span>
+hidden_size <span class="math inline">\(\times\)</span>
+intermediate_size</li>
+<li>layer norm：2 <span class="math inline">\(\times\)</span> 2 <span
+class="math inline">\(\times\)</span> hidden_size</li>
+</ul></li>
 </ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/custom_img1.png" /></p>
-<p>这篇文章是NIPS2020的工作，主要是基于知识蒸馏聚合的联邦学习。作者是来自洛桑联邦理工学院</p>
-<p>主要贡献是提出了鲁棒性的联邦蒸馏框架，并且在CV和NLP任务上都得到了验证，还对异质模型，无标签数据等进行了大量实验。</p>
-<p>右图是文章提出的算法（图是自己画的，原文并没有给方法图）,首先是根据不同的客户端在本地训练得到梯度值后，上传至服务端进行聚合，后发回给客户端进行作为教师模型进行KL散度计算，对比于之前的FedMD的方法是不加入公开数据集部分。FedDF这个方法跟之前三室的一位师姐介绍在物联网中的应用，框架基本一致。</p>
-<p>FedDF is designed for effective model fusion on the server,
-considering the accuracy of the global model on the test dataset.
-FedDF是为了有效融合模型，考虑了全局模型在测试集上的准确性，所以未对一些实验进行比较（比如FedMD）。我觉得在找借口，FedMD是有这部分实验的。</p>
-<p>文章为了辅助理解这个框架，还加入了泛化边界，数据集的分布与大小，蒸馏的实验（类似于消融实验）</p>
-<p>Lin T, Kong L, Stich S U, et al. Ensemble Distillation for Robust
-Model Fusion in Federated Learning[J]. arXiv preprint arXiv:2006.07242,
-2020.</p>
-<p>洛桑联邦理工学院</p>
-<h2 id="feddf-experiments">FedDF Experiments</h2>
-<p>2020.12.04</p>
+<p>以 <a
+href="https://huggingface.co/meta-llama/Llama-2-13b-hf/blob/main/config.json">llama2-13B</a>
+为例，关键参数</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;hidden_size&quot;</span><span class="punctuation">:</span> <span class="number">5120</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;intermediate_size&quot;</span><span class="punctuation">:</span> <span class="number">13824</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;mlp_bias&quot;</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_attention_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_hidden_layers&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_key_value_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;vocab_size&quot;</span><span class="punctuation">:</span> <span class="number">32000</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
+<p>由于 num_attention_heads == num_key_value_heads，所以是 MHA</p>
 <ul>
-<li>基本设置
+<li>Embedding/LM Head： <span class="math inline">\(32000\times
+5120\)</span></li>
+<li>transformer block
 <ul>
-<li>Local epochs(1、20、40)</li>
-<li>C(0.2、0.4、0.8)</li>
-<li>达到目标性能T（0.8、0.75）</li>
-<li>数据异质程度alpha(1、0.1)</li>
+<li>self-attention：<span class="math inline">\(4\times 5120\times
+5120\)</span></li>
+<li>FFN: <span class="math inline">\(3\times 5120\times
+13824\)</span></li>
+<li>layer norm: <span class="math inline">\(4\times 5120\)</span></li>
 </ul></li>
-<li>固定设置
+</ul>
+<p>综上，列出计算</p>
+<p><span class="math display">\[2\times 32000\times 5120 + 40\times
+(4\times 5120\times 5120 + 3\times 5120\times 13824 + 4\times 5120) =
+13,016,268,800 \approx 13B\]</span></p>
+<p>如果是 (b)float16，则占用空间/显存为 13 $$ 2 = 26G</p>
+<p>如果是 GQA，self-attn 参数量公式为</p>
+<h3 id="kv-cache-显存计算公式">KV cache 显存计算公式</h3>
+<p><span class="math inline">\(4\times b\times l\times num\_heads\times
+embed\_size\_per\_head \times (s+n)\)</span></p>
+<p>参数说明</p>
 <ul>
-<li>客户端数目：20</li>
-<li>模型架构：Resnet-8</li>
-<li>数据集：CIFAR-10</li>
-</ul></li>
-<li>对比方法：FEDAVG、FEDPROX、FEDAVGM、FEDDF</li>
+<li>b: 句子条数</li>
+<li>l：层数</li>
+<li>num_heads：隐层大小 (num_key_value_heads)</li>
+<li>embed_size_per_head：每个头的大小 (hidden_size /
+num_attention_heads)</li>
+<li>s：输入长度</li>
+<li>n：输出长度</li>
+<li>4：k cache+v cache，均为 float16，所以是(1+1)*2</li>
 </ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image4.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image5.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image6.png" /></p>
-<p>Lin T, Kong L, Stich S U, et al. Ensemble Distillation for Robust
-Model Fusion in Federated Learning[J]. arXiv preprint arXiv:2006.07242,
-2020.</p>
-<h1
-id="ensemble-distillation-for-robust-model-fusion-in-federated-learning">Ensemble
-Distillation for Robust Model Fusion in Federated Learning</h1>
-<p>2021.01.15</p>
-<h2 id="联邦学习知识蒸馏">联邦学习——知识蒸馏</h2>
+<p>如果只有1条句子，输入+输出 token 长度由 512 -&gt; 1024，则会增加
+<span class="math inline">\(4\times 512 \times l\times h\)</span>：</p>
+<p>Llama3-8B：$4 / 1024 / 1024 = 64 $ M</p>
+<p>Llama3-70B：<span class="math inline">\(4\times 512\times 1024\times
+80 / 1024 / 1024 = 160\)</span> M</p>
+<p>简单来说，每增加一个 token</p>
+<p>8B 就会增加 0.125 M 的显存</p>
+<p>70B 就会增加 0.3125 M 的显存</p>
+<h3 id="计算量">计算量</h3>
+<p>Embedding 可以视作一个哈希表，没有计算量</p>
+<ul>
+<li>LM Head：<span class="math inline">\(2 \times b \times s\times
+hidden\_size\times V\)</span></li>
+<li>Self-attention：
 <ul>
-<li>Zero-Shot（无标签）
+<li>q_proj: <span class="math inline">\(2 \times b \times s \times
+hidden\_size^2\)</span></li>
+<li>k_proj和v_proj:
 <ul>
-<li>利用公开数据集，训练GAN生成无标签的数据</li>
+<li>MHA: <span class="math inline">\(2\times(2 \times b \times s \times
+hidden\_size^2)\)</span></li>
+<li>GQA: <span class="math inline">\(2\times(2 \times b \times s \times
+hidden\_size^2 / (num\_attention\_heads
+\times  num\_key\_value\_heads)^2 )\)</span></li>
+</ul></li>
+<li>attn_weights: <span class="math inline">\(2 \times b \times s^2
+\times hidden\_size\)</span></li>
+<li>attn_output: <span class="math inline">\(2 \times b \times s^2
+\times hidden\_size\)</span></li>
+<li>o_proj: <span class="math inline">\(2 \times b \times s \times
+hidden\_size^2\)</span></li>
 </ul></li>
-<li>常规迁移学习
+<li>FFN:
 <ul>
-<li>利用客户端模型分类层权重生成数据（Dirichlet分布）</li>
+<li>gate_proj 和 up_proj: <span class="math inline">\(2\times 2\times b
+\times s\times (hidden\_size\times intermediate\_size)\)</span></li>
+<li>down_proj: <span class="math inline">\(2\times b \times s\times
+(intermediate\_size\times hidden\_size)\)</span></li>
 </ul></li>
-<li>噪声迁移学习
+</ul>
+<p>参数说明 - b: 句子条数 - s: 输入长度 - 2:
+一次乘法操作和一次加法操作</p>
+<p>还是以 <a
+href="https://huggingface.co/meta-llama/Llama-2-13b-hf/blob/main/config.json">llama2-13B</a>
+为例，关键参数</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;hidden_size&quot;</span><span class="punctuation">:</span> <span class="number">5120</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;intermediate_size&quot;</span><span class="punctuation">:</span> <span class="number">13824</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;mlp_bias&quot;</span><span class="punctuation">:</span> <span class="literal"><span class="keyword">false</span></span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_attention_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_hidden_layers&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;num_key_value_heads&quot;</span><span class="punctuation">:</span> <span class="number">40</span><span class="punctuation">,</span></span><br><span class="line"><span class="attr">&quot;vocab_size&quot;</span><span class="punctuation">:</span> <span class="number">32000</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
+<p>假设 b=1，s=1，则整体计算量有</p>
+<ul>
+<li>LM Head: <span class="math inline">\(2\times 5120\times
+32000\)</span></li>
+<li>self-attention:
 <ul>
-<li>差分隐私+联邦学习</li>
+<li><span class="math inline">\(2\times 5120\times 5120\)</span></li>
+<li><span class="math inline">\(4\times 5120\times 5120\)</span></li>
+<li><span class="math inline">\(2\times 5120\)</span></li>
+<li><span class="math inline">\(2\times 5120\)</span></li>
+<li><span class="math inline">\(2\times 5120\times 5120\)</span></li>
 </ul></li>
-<li>SWA迁移学习
+<li>MLP:
 <ul>
-<li>在一般的优化器上增加SWA，可以提高性能</li>
+<li><span class="math inline">\(4\times 5120\times 13824\)</span></li>
+<li><span class="math inline">\(2\times 13824\times 5120\)</span></li>
 </ul></li>
 </ul>
-<figure>
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image7.jpeg"
-alt="Dirichlet 分布" />
-<figcaption aria-hidden="true">Dirichlet 分布</figcaption>
-</figure>
-<figure>
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image8.png"
-alt="SWA vs SGD" />
-<figcaption aria-hidden="true">SWA vs SGD</figcaption>
-</figure>
-<p>Dirichlet分布是Beta分布的多元推广。Beta分布是二项式分布的共轭分布，Dirichlet分布是多项式分布的共轭分布。通常情况下，我们说的分布都是关于某个参数的函数，把对应的参数换成一个函数（函数也可以理解成某分布的概率密度）就变成了关于函数的函数。于是，把Dirichlet分布里面的参数换成一个基分布就变成了一个关于分布的分布了。那么它就是Dirichlet过程了。</p>
-<p>延伸阅读：<a
-href="https://www.zhihu.com/question/26751755">Dirichlet分布</a>；<a
-href="https://arxiv.org/abs/2009.05537">差分隐私+联邦学习</a>；<a
-href="https://pytorch.org/blog/stochastic-weight-averaging-in-pytorch/">SWA</a></p>
-<p>Lin T, Kong L, Stich S U, et al. Ensemble distillation for robust
-model fusion in federated learning[J]. arXiv preprint arXiv:2006.07242,
-2020.</p>
-<h1
-id="group-knowledge-transferfederated-learning-of-large-cnns-at-the-edge">Group
-Knowledge Transfer:Federated Learning of Large CNNs at the Edge</h1>
-<p>2021.04.09</p>
-<h2 id="backgroundsplit-learning">Background(Split Learning)</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image11.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image12.png" /></p>
-<h2 id="methodgroup-knowledge-transfer">Method(Group Knowledge
-Transfer)</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image9.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image10.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image13.png" /></p>
-<h1
-id="performance-optimization-for-federated-person-re-identification-via-benchmark-analysis">Performance
-Optimization for Federated Person Re-identification via Benchmark
-Analysis</h1>
-<p>2021.04.16</p>
-<ol type="1">
-<li><p>FedAVG不满足需求（不同分类），性能上也达不到（比本地训练精度低）</p></li>
-<li><p>FedPAV满足了需求，但是对于性能上不能完成满足（小数据提升，大数据集下降，训练过程振荡）</p></li>
-<li><p>FedPAV+KD满足需求，提升了性能，但是不稳定（性能不稳定，训练过程稳定）</p>
-<ol type="1">
-<li>Tricks：正则化</li>
-<li>Tricks：Cosing Distance Weight</li>
-</ol></li>
-</ol>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image14.png" /></p>
-<h1
-id="fedbn-federated-learning-on-non-iid-features-via-local-batch-normalization">FedBN:
-Federated Learning on Non-IID Features via Local Batch
-Normalization</h1>
-<p>2021.04.23</p>
-<h2 id="实验设计分析">实验设计分析</h2>
-<ol type="1">
-<li>选定一个数据集，比较：本地训练轮数E，本地数据集大小，客户端数目（切割数据集数量）（左上角三张图）
-<ul>
-<li>可说明：收敛速度、E的影响，数据集大小影响，异质性影响</li>
-</ul></li>
-<li>不同数据集进行比较。说明有效果（右上角一张图）</li>
-<li>基于2，有效果再比较其他三种不同领域的任务。实验设置+实验分析（简要）</li>
-</ol>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image15.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image16.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image17.png" /></p>
-<h1 id="adaptive-federated-optimization">Adaptive Federated
-Optimization</h1>
-<p>2021.05.14</p>
-<h2 id="problem">Problem</h2>
-<p>Standard federated optimization methods such as Federated Averaging
-(FedAvg) are often difficult to tune and exhibit unfavorable convergence
-behavior.</p>
-<h2 id="inspired">Inspired</h2>
-<p>In non-federated settings, <strong>adaptive optimization</strong>
-methods have had notable success in combating such issues.</p>
-<h2 id="work">Work</h2>
-<p>In this work, we propose <strong>federated versions of adaptive
-optimizers</strong>, including Adagrad, Adam, and Yogi, and analyze
-their convergence in the presence of heterogeneous data for general
-nonconvex settings.</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image18.png" /></p>
-<p>Google: 2020 Feb-2020 Dec</p>
-<h1
-id="preservation-of-the-global-knowledge-by-not-true-self-knowledge-distillation-in-federated-learning">Preservation
-of the Global Knowledge by Not-True Self Knowledge Distillation in
-Federated Learning</h1>
-<p>2021.06.18</p>
-<h2 id="动机">动机</h2>
-<p>Catastrophic Forgetting: feature shifting induced fitting on biased
-local</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image19.png" /></p>
-<h2 id="研究">研究</h2>
-<p>训练方式: 本地自蒸馏(LSD)</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image20.png" /></p>
-<p>二次改进: Not-True Distillation</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image21.png" /></p>
-<p>有偏局部特征转移诱导拟合</p>
-<p>遗忘灾难：把CIFAR10输入拉成二维，在进行100轮联邦学习后，估计数据的概率密度函数，上图这个分布存在明显偏差；提出的方法基本无偏差</p>
-<p>研究方法：将传统的训练，改成了蒸馏的方式（由于是模型架构相同，所以可以叫做自蒸馏），也在本地进行蒸馏；另外，基于IJCAJ2021知识蒸馏的工作，对教师模型预测正确的标签剔除，再KL散度计算，以进行改进。</p>
-<p>Taehyeon Kim, Jaehoon Oh, NakYil Kim, Sangwook Cho, and Se-Young Yun.
-Comparingkullback-leibler divergence and mean squared error loss in
-knowledge distillation.arXiv preprintarXiv:2105.08919, 2021.</p>
-<h1
-id="preservation-of-the-global-knowledge-by-not-trueself-knowledge-distillation-in-federated-learning">Preservation
-of the Global Knowledge by Not-TrueSelf Knowledge Distillation in
-Federated Learning</h1>
-<p>2021.06.25</p>
-<p>利用知识蒸馏保留旧任务的知识</p>
-<p><span
-class="math display">\[{\mathcal{L}}_{\mathrm{FedLSD}}=(1-\beta)\cdot{\mathcal{L}}_{\mathrm{CE}}(q,\;p_{y})+\beta\cdot{\mathcal{L}}_{\mathrm{LSD}}(q_{\tau},\;q_{\tau}^{y})\quad(0&lt;\beta&lt;1)\;\]</span></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image23.png" /></p>
-<p>Weight divergence</p>
-<p><span class="math display">\[\rm{weight\,divergence}=||w^{F e d A v
-g}-w^{S G D}||/||w^{S G D}||\]</span></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image22.png" /></p>
-<p>全局模型的预测可以作为先前数据分布的参考，从而产生类似于在CL中使用情景记忆的效果。</p>
-<p>beta是指KL散度损失的权重</p>
-<p>结果表明，数据分布的正确性，即数据分布的偏态性，可能影响数据的准确性。</p>
-<p>Zhao Y, Li M, Lai L, et al. Federated learning with non-iid data[J].
-arXiv preprint arXiv:1806.00582, 2018.</p>
-<h1 id="learn-distributed-gan-with-temporary-discriminators">Learn
-distributed GAN with Temporary Discriminators</h1>
-<p>2021.07.02</p>
-<p>研究动机：医疗领域，利用本地的隐私数据进行数据增强</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image26.png" /></p>
-<p><a
-href="https://github.com/huiqu18/TDGAN-PyTorch">https://github.com/huiqu18/TDGAN-PyTorch</a></p>
-<h1
-id="personalized-federated-learning-with-theoretical-guarantees-a-model-agnostic-meta-learning-approach">Personalized
-Federated Learning with Theoretical Guarantees: A Model-Agnostic
-Meta-Learning Approach</h1>
-<p>2021.07.16</p>
-<h2 id="动机-1">动机</h2>
-<p>联邦学习+元学习-&gt;个性化联邦学习(Per-FedAVG) <img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image29.png" /></p>
-<h2 id="元学习maml">元学习——MAML</h2>
-<p><span class="math display">\[w_{k+1}\leftarrow
-w_{k}-(\beta_{k}/B)\sum_{i\in
-B_{k}}\bar{\nabla}f_{i}(w_{k+1}^{i},\mathcal{D}_{o}^{i})\]</span></p>
-<h2 id="first-order">First-Order</h2>
-<p><span
-class="math display">\[w_{k+1}=w_{k}-{\frac{\beta_{k}}{\mathcal{B}}}\sum_{i\in{B_{k}}}\left[\tilde{\nabla}f_{i}\Bigl(w_{k}-\alpha\tilde{\nabla}f_{i}(w_{k},D_{i
-n}^{i}),D_{o}^{i}\right)-\alpha d_{k}^{i}]\]</span></p>
-<h2 id="hessian-free">Hessian-Free</h2>
-<p><span
-class="math display">\[w_{k+1}=w_{k}-\beta_{k}{\frac{1}{B}}\sum_{i\in{\mathcal{B}_{k}}}\left(I-\alpha\tilde{\nabla}^{2}f_{i}(w_{k},\mathcal{D}_{h}^{i})\right)\tilde{\nabla}f_{i}(w_{k+1}^{i},D_{o}^{i})\]</span></p>
-<p>这篇论文是发表在NIPS2020，是关于个性化联邦学习的文章。作者来自于MIT的LIDS团队。文章主要是通过统计优化角度的层面对问题进行分析，并给予理论分析与实验结果证明方法的有效性。</p>
-<p>从元学习的角度来分析这篇文章，作者是同一批人。他们首先提出了MAML这种元学习的方法，右侧图，w
-的hat，是提出的问题建模。从w
-hat开始训练，更容易适用于大部分任务。即原始公式</p>
-<p>根据原始公式，得到两种近似，一阶形式，与海森形式（二阶）。</p>
-<p>Alireza Fallah, Aryan Mokhtari, Asuman Ozdaglar. Personalized
-Federated Learning with Theoretical Guarantees: A Model-Agnostic
-Meta-Learning Approach. In Advances in Neural Information Processing
-Systems (NIPS), 2020(33): 3557–3568.</p>
-<h1 id="distilled-one-shot-federated-learning">Distilled One-Shot
-Federated Learning</h1>
-<p>2021.08.20</p>
-<h2 id="动机-2">动机</h2>
-<p>Solve the communication challenges of FL</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image31.png" /></p>
-<h2 id="方法">方法</h2>
-<ul>
-<li>Clients: datasets distillation</li>
-<li>Send distilled data</li>
-<li>Server train
-<ul>
-<li>Parallel: hard resets + randomly adjusts (slow)</li>
-<li>Serial: one by one distillated ## 实验</li>
-</ul></li>
-<li>Image Classification
-<ul>
-<li>MNIST: LeNet</li>
-</ul></li>
-<li>Text Classification
+<p>模型有 40 层，所以</p>
+<p><span class="math inline">\(2\times 5120\times 32000+
+40\times(8\times 5120\times 5120+4\times 5120+ 2\times 5120\times
+13824+2\times 13824\times 5120) = 20,041,728,000\)</span></p>
+<p>参考资料：</p>
+<p>https://blog.csdn.net/wxc971231/article/details/135434478</p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>NLP</tag>
+        <tag>Inference</tag>
+        <tag>Math</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>LLM的推理加速</title>
+    <url>/hexo_blog/2024/06/15/LLM%E7%9A%84%E6%8E%A8%E7%90%86%E5%8A%A0%E9%80%9F/</url>
+    <content><![CDATA[<p>对于两种场景下介绍 LLM 的 推理加速方法</p>
+<p>长输入，短输出</p>
+<p>长输出但输出 token 有限</p>
+<span id="more"></span>
+<p>大语言模型在进行推理时是非常耗时的，因为需要对每个 token
+进行计算。对于推理加速这个话题，在工程上可以从很多方面考虑：</p>
 <ul>
-<li>IMDB: TextCNN</li>
-<li>TREC-6: Bi-LSTM</li>
-<li>SENT140: TextCNN</li>
-</ul></li>
+<li>模型进行量化/剪枝，降低模型浮点计算成本</li>
+<li>Continuous Batch，批量计算 Next Token</li>
+<li>KV Cache：LLM 推理的是 Token by Token
+生成，每次生成的时候会计算所有输入 Token 的
+Attention，故可以缓存已计算的 Attention，减少计算量</li>
+<li>对显存进行管理，如 <a
+href="https://wnma3mz.github.io/hexo_blog/2023/11/24/vLLM%E7%9A%84PageAttention/">PageAttention</a></li>
+<li>KV Cache 量化</li>
+<li>...</li>
 </ul>
-<p>由于有大量蒸馏的数据，不能直接进行聚合。所以有串行与并行两种解决方案。</p>
-<p>数据蒸馏：不能在没有resets的模型中直接训练，否则无法达到较好的精度</p>
-<p>hard resets: 强制初始化模型参数</p>
-<p>randomly adjusts: 在每次学习过程中，进行随机的调整</p>
-<p>Random masking randomly selects a fraction p_{rm} of the distilled
-data at each training iteration andreplaces it with a random tensor.</p>
-<p>Zhou Y, Pu G, Ma X, et al. Distilled one-shot federated learning[J].
-arXiv preprint arXiv:2009.07999, 2020.</p>
-<p>NSF Center for Big Learning University of Florida</p>
-<h1 id="does-knowledge-distillation-really-work">Does Knowledge
-Distillation Really Work?</h1>
-<p>2021.08.27</p>
-<h2 id="agreement-and-fidelity">Agreement and Fidelity</h2>
-<p><span
-class="math display">\[{\mathrm{Average~Predictive~KL}}:=\frac{1}{n}\sum_{i=1}^{n}{\mathrm{KL}}\left(\hat{p}_{t}({\bf
-y}|{\bf x}_{i})\mid\right|\hat{p}_{s}({\bf y}|{\bf x}_{i}))\]</span></p>
-<h2 id="generalization-vs-fidelity">Generalization vs Fidelity</h2>
+<p>而其他在工程上的优化，往往是针对特定场景的，下面介绍两种场景下的优化方法。</p>
+<h2 id="prefix-cache长输入短输出">Prefix Cache：长输入，短输出</h2>
+<p>注：优化显存并不会减少计算量，只是减少了显存占用，从而提升了吞吐。</p>
+<h3 id="kv-cache-显存计算公式">KV cache 显存计算公式</h3>
+<p><span class="math inline">\(4\times b\times l\times num\_heads\times
+embed\_size\_per\_head \times (s+n)\)</span></p>
+<p>参数说明</p>
 <ul>
-<li>SD: Data ⬆ Fidelity ⬆, Accuracy ⬇</li>
-<li>ED: Data ⬆ Fidelity ⬆, Accuracy ⬆</li>
+<li>b: 句子条数</li>
+<li>l：层数</li>
+<li>num_heads：隐层大小 (num_key_value_heads)</li>
+<li>embed_size_per_head：每个头的大小 (hidden_size /
+num_attention_heads)</li>
+<li>s：输入长度</li>
+<li>n：输出长度</li>
+<li>4：k cache+v cache，均为 float16，所以是(1+1)*2</li>
 </ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image33.png" /></p>
-<h2 id="why-low-fidelity">Why Low Fidelity</h2>
+<p>如果只有1条句子，输入+输出 token 长度由 512 -&gt; 1024，则会增加
+<span class="math inline">\(4\times 512 \times l\times h\)</span>：</p>
+<p>Llama3-8B：$4 / 1024 / 1024 = 64 $ M</p>
+<p>Llama3-70B：<span class="math inline">\(4\times 512\times 1024\times
+80 / 1024 / 1024 = 160\)</span> M</p>
+<p>简单来说，每增加一个 token</p>
+<p>8B 就会增加 0.125 M 的显存</p>
+<p>70B 就会增加 0.3125 M 的显存</p>
+<p>这个显存占有直到整个句子生成完成之后才会释放，所以对于长句子，KV
+Cache 的显存占用是非常大的。</p>
+<p>显存决定了整体的吞吐量。对于 70B，假设有 1k 个请求，每个请求的总
+token 为 1k，那么显存占用为 0.3125M * 1000 * 1000 = 0.3G。</p>
+<h3 id="方法">方法</h3>
+<p>在 LLM
+推理时，往往是用对话的方式进行推理，而每次对话的开头可能是相同（System），这一部分的
+Attention 是不变的，所以可以缓存这部分 Attention。</p>
+<p>加速的原因：</p>
 <ul>
-<li>Architecture: ResNet √ ; VGG ×</li>
-<li>Student Capacity: ×</li>
-<li>Identifiability：distillation dataset ≠ test dataset</li>
-<li>Optimization：√ <img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image32.png" /></li>
+<li>prefix 的 kv cache 不需要计算</li>
+<li>prefix 的 kv cache 复用节约了显存，提升吞吐（主要收益）</li>
 </ul>
-<p>知识蒸馏在深度学习中被广泛应用，但其是否真的有效？简单来说，KD确实有效，能够提升学生网络模型的性能；然而教师往往只能传递有限的知识给学生。
-文章提出了一个新的名词Agreement，或者说Fidelity，即教师与学生对于数据的预测相似度。</p>
-<p>为什么要提出一个新的概念，根据图1中的实验结果xxx</p>
-<p>为什么会有更低的保真度 1. 优化器。随着epoch的增加，一致性增加 2.
-初始化。教师权重+随机权重，lambda比值，越大表示趋近于教师</p>
-<p>我们发现，如果学生的初始化距离教师很远（λ≤0.25），优化器会收敛到蒸馏损失的次优值，从而产生与教师明显不同的学生。然而在λ=
-0.375
-处有一个突然的变化。最终的训练损失下降到最优值并且一致性急剧增加，并且行为继续
-λ &gt;0.375。为了进一步研究，在图 6 (c) 中，我们将 λ∈ {0,0.25,0.375}
-的蒸馏损失表面可视化，投影在与 θt
-相交的二维子空间、初始学生权重和最终学生权重上。如果学生初始化离老师很远（λ∈{0,0.25}），它会收敛到损失表面的一个不同的、次优的盆地。另一方面，当初始化接近于老师（λ=
-0.375）时，学生收敛到与老师相同的盆地，达到接近 100% 的一致性</p>
-<p>我们终于确定了之前所有干预措施无效的根本原因。知识蒸馏无法收敛到最佳学生参数，即使我们知道一个解决方案并在优化方向上给初始化一个小的开端。事实上，虽然可识别性可能是一个问题，但为了在所有输入上匹配教师，学生有
-至少在用于蒸馏的数据上匹配老师，并实现蒸馏损失的接近最优值。在实践中，优化收敛到次优解，导致蒸馏保真度不佳</p>
-<p>Stanton S, Izmailov P, Kirichenko P, et al. Does Knowledge
-Distillation Really Work?[J]. arXiv preprint arXiv:2106.05945, 2021.</p>
-<p>Google Research</p>
-<h1 id="model-contrastive-federated-learning">Model-Contrastive
-Federated Learning</h1>
-<p>2021.09.18</p>
-<h2 id="motivation-1">Motivation</h2>
-<p>A key challenge in federated learning is to handle the heterogeneity
-of local data distribution across parties Method</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image35.png" /></p>
-<h2 id="method-1">Method</h2>
-<p><span class="math display">\[\ell_{c o
-n}=-\log\frac{\exp(\mathrm{sim}(z,z_{g l o
-b})/\tau)}{\exp(\mathrm{sim}(z,z_{g l o
-b})/\tau)+\exp(\mathrm{sim}(z,z_{p r e e v})/\tau)}\]</span></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image36.png" /></p>
-<p>Li Q, He B, Song D. Model-Contrastive Federated
-Learning[C]//Proceedings of the IEEE/CVF Conference on Computer Vision
-and Pattern Recognition. 2021: 10713-10722.</p>
-<p>UC Berkeley</p>
-<h1
-id="data-free-knowledge-distillation-for-heterogeneous-federated-learning">Data-Free
-Knowledge Distillation for Heterogeneous Federated Learning</h1>
-<p>2021.09.27</p>
-<h2 id="motivation-2">Motivation</h2>
-<p>The ensemble knowledge is not fully utilized to guide local model
-learning, which mayin turn affect the quality of the aggregated
-model.</p>
-<h2 id="method-2">Method</h2>
-<p><span
-class="math display">\[\operatorname*{min}_{\theta}\operatorname*{lim}_{x\rightarrow\hat{\mathcal{D}}_{\mathrm{p}}}\left[D_{\mathrm{KL}}[\sigma(\frac{1}{K}\sum_{k=1}^{K}g(f(x;\theta_{k}^{f});\theta_{k}^{p})]|\sigma(g(f(x;\theta^{f});\theta^{p})]\right].\]</span></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image38.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image40.png" /></p>
-<p>Zhuangdi Zhu and Junyuan Hong and Jiayu Zhou. 2021. Data-Free
-Knowledge Distillation for Heterogeneous Federated Learning. In ICML,
-139:12878-12889.</p>
-<p>Michigan State University</p>
-<h1 id="beyond-sharing-weights-for-deep-domain-adaptation">Beyond
-Sharing Weights for Deep Domain Adaptation</h1>
-<p>2021.10.18</p>
-<h2 id="motivation-3">Motivation</h2>
-<p>Learn features that are invariant to the domain shift.</p>
-<h2 id="method-3">Method</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image41.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image43.png" /></p>
-<h2 id="discussion">Discussion</h2>
-<p>It therefore seems reasonable that thehigher layers of the network,
-which encode higher-level in-formation, should be domain-specific.</p>
-<p>Rozantsev, A., Salzmann, M., &amp; Fua, P. (2019). Beyond Sharing
-Weights for Deep Domain Adaptation. IEEE Transactions on Pattern
-Analysis and Machine Intelligence, 41(4), 801-814.</p>
-<h1
-id="feddg-federated-domain-generalization-on-medical-image-segmentation-via-episodic-learning-in-continuous-frequency-space">FedDG:
-Federated Domain Generalization on Medical Image Segmentation via
-Episodic Learning in Continuous Frequency Space</h1>
-<p>2021.10.25</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image50.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image48.png" /></p>
-<h2 id="method-4">Method</h2>
-<ul>
-<li>Data
+<p>所以适用场景的一个明显特点就是必须要有长的 system prefix，而且这个
+prefix
+是不变的。另外，如果模型输出过长的话，这个优化的收益就会变小。因为它并不会加速在生成过程中的计算，只是减少了显存占用。</p>
+<h3 id="收益">收益</h3>
+<p>https://github.com/vllm-project/vllm/issues/227</p>
+<p>For each request, the prefix length is 200, the input length is 30,
+and the output length is 50.</p>
+<table>
+<colgroup>
+<col style="width: 12%" />
+<col style="width: 15%" />
+<col style="width: 20%" />
+<col style="width: 29%" />
+<col style="width: 20%" />
+</colgroup>
+<thead>
+<tr>
+<th>Load (QPS)</th>
+<th>Method</th>
+<th>Requests/s</th>
+<th>Average Latency per Req</th>
+<th>First Token Time</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>10 QPS</td>
+<td>Prefix Cache</td>
+<td>9.83 requests/s</td>
+<td>1.97 s</td>
+<td>0.29 s</td>
+</tr>
+<tr>
+<td>10 QPS</td>
+<td>Base</td>
+<td>9.80 requests/s</td>
+<td>2.87 s</td>
+<td>0.45 s</td>
+</tr>
+<tr>
+<td>15 QPS</td>
+<td>Prefix Cache</td>
+<td>14.30 requests/s</td>
+<td>2.98 s</td>
+<td>0.39 s</td>
+</tr>
+<tr>
+<td>15 QPS</td>
+<td>Base</td>
+<td>13.24 requests/s</td>
+<td>8.65 s</td>
+<td>1.02 s</td>
+</tr>
+<tr>
+<td>25 QPS</td>
+<td>Prefix Cache</td>
+<td>19.81 requests/s</td>
+<td>6.46 s</td>
+<td>0.84 s</td>
+</tr>
+<tr>
+<td>25 QPS</td>
+<td>Base</td>
+<td>14.08 requests/s</td>
+<td>13.67 s</td>
+<td>4.74 s</td>
+</tr>
+</tbody>
+</table>
+<h2 id="并行解码长输出但输出-token-有限">并行解码：长输出但输出 token
+有限</h2>
+<p>LLM 的推理是 Token by Token 生成的，如果有一种方式可以每次生成多个
+token，那么就可以提升推理速度。</p>
+<h3 id="方法-1">方法</h3>
+<p>以下图为例，对 LLM 的模型结构进行一定的修改，使每次可以同时生成 3 个
+token。</p>
+<figure>
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/llm_speedup/251e873bb1322b4d31fee19f0c81c424.png"
+alt="并行解码" />
+<figcaption aria-hidden="true">并行解码</figcaption>
+</figure>
+<p>并行解码的方式不一定要对结构进行修改，这里只是一种实现方式。</p>
+<p>具体实现方式： -
+训练：修改结构后对模型的新增部分进行常规的训练，其他部分冻结，减少训练成本
+- 推理：模型生成的时候会把连续 3 个 token 的 top-K
+的可能均生成，交由原模型进行打分，然后选择最优的一个。</p>
+<h3 id="问题">问题</h3>
+<p>并行生成的 Token 的可靠性是取决于具体场景的。如果生成的 Token
+可能性只有 20 个，那么并行生成的 Token
+的可靠性就会相对较高，这种对于模型而言是更容易学习的。但如果生成的 Token
+可能有 10000 个，那么并行生成的 Token 的可靠性就会相对较低。</p>
+<p>换而言之，Token
+可能性越少越容易猜中，越多越难猜中。所以，这种方法适用于输出 token
+可能性有限的场景。</p>
+<p>并且，如果输出 token 数量较短，那么相较于 token by token
+的生成，这种方法的优势就会变小。</p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>NLP</tag>
+        <tag>Attention</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>SPSA随机优化算法</title>
+    <url>/hexo_blog/2023/08/19/SPSA%E9%9A%8F%E6%9C%BA%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95/</url>
+    <content><![CDATA[<p>SPSA(Simultaneous Perturbation Stochastic
+Approximation)是一种随机优化算法,可用于神经网络的训练。其基于随机梯度来逼近真实梯度。无法直接得到精确梯度,收敛速度较慢,但节省计算资源。</p>
+<span id="more"></span>
+<h3 id="spsa算法简介">SPSA算法简介</h3>
+<p>SPSA是一种随机优化算法,可以用于神经网络模型的训练。它的主要思想是同时随机扰动所有参数,基于损失函数在扰动后的变化来估计梯度。</p>
+<p>相比确定性的梯度下降法,SPSA依靠随机性搜索最优解,可以更好地跳出局部最优,具有全局搜索能力。另外,SPSA只需要前向传播就可以更新参数,省去了存储反向传播所需的计算图,计算开销较小。</p>
+<h3 id="pytorch实现spsa训练神经网络">PyTorch实现SPSA训练神经网络</h3>
+<p>要在PyTorch中实现SPSA算法训练神经网络,主要分以下几个步骤:</p>
+<ol type="1">
+<li><p>构建神经网络模型</p>
+<p>可以构建一个多层全连接网络,例如在MNIST数据集上进行图像分类任务:</p>
+<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch.nn <span class="keyword">as</span> nn</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Net</span>(nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="built_in">super</span>(Net, <span class="variable language_">self</span>).__init__()</span><br><span class="line">        <span class="variable language_">self</span>.fc1 = nn.Linear(<span class="number">784</span>, <span class="number">100</span>)</span><br><span class="line">        <span class="variable language_">self</span>.fc2 = nn.Linear(<span class="number">100</span>, <span class="number">10</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line">        x = F.relu(<span class="variable language_">self</span>.fc1(x))</span><br><span class="line">        x = <span class="variable language_">self</span>.fc2(x)</span><br><span class="line">        <span class="keyword">return</span> F.log_softmax(x, dim=<span class="number">1</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure></p></li>
+<li><p>定义损失函数和优化器</p>
+<p>使用交叉熵损失函数和SGD优化器:</p>
+<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line">criterion = nn.CrossEntropyLoss()</span><br><span class="line">optimizer = torch.optim.SGD(net.parameters(), lr=<span class="number">0.1</span>)</span><br></pre></td></tr></table></figure></p></li>
+<li><p>实现SPSA算法</p>
+<p>根据SPSA的公式,可以实现计算随机梯度的函数:</p>
+<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">SPSA_step</span>(<span class="params">net, criterion, x, y, a, c, k</span>):</span><br><span class="line">    delta = <span class="number">2</span> * (torch.randint(<span class="number">0</span>, <span class="number">2</span>, net.parameters().__len__()) - <span class="number">0.5</span>)</span><br><span class="line">    perturbation = c * delta</span><br><span class="line">    loss1 = criterion(net(x + a*perturbation), y)</span><br><span class="line">    loss2 = criterion(net(x - a*perturbation), y)</span><br><span class="line">    gradient = (loss1 - loss2) / (<span class="number">2</span> * a * perturbation)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> param, grad <span class="keyword">in</span> <span class="built_in">zip</span>(net.parameters(), gradient):</span><br><span class="line">        param.data.add_(k * grad * delta)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> net</span><br></pre></td></tr></table></figure></p></li>
+<li><p>训练神经网络</p>
+<p>在训练循环中调用SPSA_step函数计算梯度并更新参数:</p>
+<p><figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">    running_loss = <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> i, data <span class="keyword">in</span> <span class="built_in">enumerate</span>(trainloader, <span class="number">0</span>):</span><br><span class="line">        inputs, labels = data</span><br><span class="line">        </span><br><span class="line"></span><br><span class="line">        net = SPSA_step(net, criterion, inputs, labels, <span class="number">0.01</span>, <span class="number">0.005</span>, <span class="number">0.01</span>)</span><br><span class="line">				optimizer.zero_grad()</span><br><span class="line">        optimizer.step()</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;[%d] loss: %.3f&#x27;</span> % (epoch + <span class="number">1</span>, running_loss / <span class="built_in">len</span>(trainloader)))</span><br><span class="line"></span><br></pre></td></tr></table></figure></p></li>
+</ol>
+<h3 id="前向传播与反向传播的比较">前向传播与反向传播的比较</h3>
+<p>使用前向传播计算梯度时,需要使用两点式差分来估计梯度:</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">gradient = (loss1 - loss2) / (<span class="number">2</span> * a * perturbation)</span><br></pre></td></tr></table></figure>
+<p>其中loss1和loss2分别对应参数做微小正负扰动时的损失。这只能得到梯度的近似值,收敛速度较慢。</p>
+<p>而反向传播可以通过动态计算图高效地得到每个参数的精确梯度,使得网络可以更快收敛。但需要额外的存储空间保存计算图,计算复杂度也更高。</p>
+<p>所以在训练神经网络时,通常选择反向传播来获取精确梯度信息,以保证收敛速度。SPSA更适用于一些计算资源受限但需要求解最优参数的场景。</p>
+<h3 id="q-a">Q &amp; A</h3>
+<p><strong>问题1:SPSA算法与随机梯度下降(SGD)的区别是什么?</strong></p>
+<p>SPSA和SGD都依赖于随机性来达到优化目的,主要区别在于:</p>
 <ul>
-<li>Step 1：对每个客户端的数据都做如右处理</li>
-<li>Step
-2：客户端k的图像X，与傅里叶变换后的数据A（联合任意两个客户端的A）
+<li>SGD每次只随机扰动一个参数,SPSA同时扰动所有参数</li>
+<li>SPSA使用有限差分法估计梯度,SGD使用反向传播计算精确梯度</li>
+<li>SPSA可能更能避免局部最优,SGD收敛速度更快</li>
+</ul>
+<p>使用前向传播和反向传播的主要不同是在计算梯度的方式上。</p>
+<p>使用反向传播时，我们首先通过前向传播计算网络的输出，然后将输出与实际标签计算损失函数，然后通过反向传播计算每个参数对损失函数的梯度。这使得我们能够有效地更新模型参数，并最小化损失函数。</p>
+<p>而如果只使用前向传播进行神经网络的更新，则无法直接计算每个参数对损失函数的梯度。相反，我们可以考虑使用“两点式差分”来近似计算梯度。具体地，我们采用以下公式：</p>
+<p><span class="math display">\[\nabla f(x) \approx \frac{f(x+\Delta x)
+- f(x-\Delta x)}{2\Delta x}\]</span></p>
+<p>其中，<span class="math inline">\(\Delta
+x\)</span>代表一个微小的扰动，<span
+class="math inline">\(x\)</span>是输入数据，<span
+class="math inline">\(f(x)\)</span>为损失函数。我们根据这个公式对每个参数进行微小的扰动，然后计算对应的损失函数。在此基础上，我们可以使用公式计算梯度并更新参数。</p>
+<p>使用前向传播进行更新相比使用反向传播的优势在于，它减少了内存占用和计算时间，并且不需要存储计算图。但是，由于它只是使用一阶近似来计算梯度，因此可能会导致更新不如反向传播精确。</p>
+<p><strong>问题2:SPSA算法适用于哪些场景?</strong></p>
+<p>SPSA算法适用于以下场景:</p>
 <ul>
-<li>对X进行傅里叶变换</li>
-<li>把X的振幅分量中的低频分量换为A的低频分量， low_freq_mutate</li>
-<li>最后结合A，做逆傅里叶变换，得到新的图像X’</li>
-</ul></li>
-</ul></li>
-<li>Train
+<li>目标函数无法明确表达,无法求导</li>
+<li>参数空间高维,难以精确计算梯度</li>
+<li>需要避免陷入局部最优</li>
+<li>计算资源受限,无法存储大规模计算图</li>
+</ul>
+<p><strong>问题3:SPSA算法的两大超参数a和c表示什么?</strong></p>
 <ul>
-<li>Loss1:
-根据原始图像X做训练。先利用这里的梯度计算梯度下降后的模型参数F’</li>
-<li>Loss2: 根据F’对X’的输出计算</li>
-<li>Loss3: F与F’ 分别提取mask的边界与背景
-(ndimage.binary_erosion)做对比学习NTXentLoss，loss3*0.1</li>
-</ul></li>
+<li>a是步长(perturbation size),表示参数扰动大小</li>
+<li>c是抖动幅度(perturbation distribution),控制扰动值波动范围</li>
 </ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image51.png" /></p>
-<p>解决从多个分布域中学习的问题-&gt;数据异质问题</p>
-<p>一般的深度学习中，数据是共享的，所以可以直接进行跨域学习。然而在联邦学习中，数据并不支持直接共享，所以从数据分布角度入手。从原始图像中抽取中分布(风格)信息，并把这个信息共享出来。这里假设通过这个分布信息无法还原真实数据。How？</p>
-<p>通过傅里叶变换，把数据转换为：振幅谱+相位谱。下图的蓝色背景部分，通过公式进行转换后，再把各个客户端的信息数据进行汇总学习。</p>
-<p>由于训练背景是医学图像处理，涉及到语义分割，所以需要使用常规的分割损失函数。再考虑图形的边界与背景，对应右边的绿色部分，采用InfoNCE损失函数
-对边界与背景做提取处理，计算InfoNCE损失。hm，hp表示同一类的正面特征，表示边界与背景。</p>
-<p>所以整个训练过程就是利用傅里叶变换提取分布信息，结合一般的分割与InfoNCE损失进行训练。主要的创新点在傅里叶变换提取了可共享的信息。下一次再进行具体的方法介绍。</p>
-<p>Liu Q, Chen C, Qin J, et al. Feddg: Federated domain generalization
-on medical image segmentation via episodic learning in continuous
-frequency space. Proceedings of the IEEE/CVF Conference on Computer
-Vision and Pattern Recognition. 2021: 1013-1023.</p>
-<p>CUHK, 香港中文大学</p>
-<h1
-id="refine-myself-by-teaching-myself-feature-refinement-via-self-knowledge-distillation">Refine
-Myself by Teaching Myself : Feature Refinement via Self-Knowledge
-Distillation</h1>
-<p>2021.11.11</p>
-<h2 id="method-5">Method</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image53.png" /></p>
-<p><span class="math display">\[
-\begin{array}{l}
-\mathcal{L}_{FRSKD}(\bm{x},y;\theta_{c},\theta_{t},K) =
-\mathcal{L}_{CE}(\bm{x},y;\theta_{c})+\mathcal{L}_{C
-E}(\bm{x},y;\theta_{t}) +\alpha\cdot\mathcal{L}_{K
-D}(\bm{x};\theta_{c},\theta_{t},K)+\beta\cdot\mathcal{L}_{F}(T,F;\theta_{c},\theta_{t})
-\end{array}
-\]</span></p>
-<p><span
-class="math display">\[\mathcal{L}_{F}(T,F;\theta_{c},\theta_{t})=\Sigma_{i=1}^{n}\vert\vert\phi(T_{i})-\phi(F_{i})\vert\vert_2\]</span></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image56.png" /></p>
-<h2 id="result-1">Result</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image57.png" /></p>
-<p><a
-href="https://github.com/MingiJi/FRSKD">https://github.com/MingiJi/FRSKD</a></p>
-<p>Ji M, Shin S, Hwang S, et al. Refine Myself by Teaching Myself:
-Feature Refinement via Self-Knowledge Distillation. Proceedings of the
-IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2021:
-10664-10673.</p>
-<h1
-id="no-fear-of-heterogeneity-classifier-calibration-for-federated-learning-with-non-iid-data">No
-Fear of Heterogeneity: Classifier Calibration for Federated Learning
-with Non-IID Data</h1>
-<p>2021.11.25</p>
-<h2 id="motivation-4">Motivation</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image58.png" /></p>
-<h2 id="method-6">Method</h2>
-<p><span
-class="math display">\[\mu_{c,k}=\frac{1}{N_{c,k}}\sum_{j=1}^{N_{c,k}}z_{c,k,j},\quad\Sigma_{c,k}=\frac{1}{N_{c,k}-1}\sum_{j=1}^{N_{c,k}}\left(z_{c,k,j}-\mu_{c,k}\right)\left(z_{c,k,j}-\mu_{c,k}\right)^{T}\]</span></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image61.png" /></p>
-<h2 id="result-2">Result</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image62.png" /></p>
-<p>Luo M, Chen F, Hu D, et al. No Fear of Heterogeneity: Classifier
-Calibration for Federated Learning with Non-IID Data[J]. arXiv preprint
-arXiv:2106.05001, 2021.</p>
-<h1 id="federated-learning-with-personalization-layers">Federated
-Learning with Personalization Layers</h1>
-<p>2021.12.02</p>
-<h2 id="method-7">Method</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image64.png" /></p>
-<p>Personalized population risk</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image65.png" /></p>
-<p>Empirical risk</p>
-<p><span class="math display">\[L_{j}^{E
-R}\big(\displaystyle\mathrm{W}_B,\displaystyle\mathrm{W}_{P}\big)\triangleq\frac{1}{n_{j}}\sum_{i=1}^{n_{j}}l(y_{j,i},f\big(x_{j,i};\displaystyle\mathrm{W}_B,\displaystyle\mathrm{W}_{P}\big)\big)\]</span></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image67.png" /></p>
-<p>Manoj Ghuhan Arivazhagan, Vinay Aggarwal, Aaditya Kumar Singh, and
-Sunav Choudhary. 2019. Federated learning with personalization layers.
-arXiv preprint arXiv:1912.00818 (2019).</p>
-<h1
-id="federated-split-vision-transformer-for-covid-19-cxr-diagnosis-using-task-agnostic-training">Federated
-Split Vision Transformer for COVID-19 CXR Diagnosis using Task-Agnostic
-Training</h1>
-<p>2021.12.09</p>
-<h2 id="motivation-5">Motivation</h2>
-<p>融合联邦学习与分割学习尽可能发挥他们的独特优势</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image69.png" /></p>
-<h2 id="method-8">Method</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image70.png" /></p>
-<p>Park S, Kim G, Kim J, et al. Federated Split Vision Transformer for
-COVID-19 CXR Diagnosis using Task-Agnostic Training[J]. arXiv preprint
-arXiv:2111.01338, 2021.</p>
-<h1
-id="local-learning-matters-rethinking-data-heterogeneity-in-federated-learning">Local
-Learning Matters: Rethinking Data Heterogeneity in Federated
-Learning</h1>
-<p>2022.03.18</p>
-<h2 id="contribution">Contribution</h2>
-<p>Focus on local learning generality rather than proximal
-restriction</p>
-<h2 id="method-9">Method</h2>
-<h3 id="stochastic-depth-2016">Stochastic Depth (2016)</h3>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image71.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> <span class="keyword">not</span> <span class="variable language_">self</span>.training <span class="keyword">or</span> torch.rand(<span class="number">1</span>)[<span class="number">0</span>] &gt;= <span class="variable language_">self</span>.death_rate:</span><br><span class="line">    ……</span><br><span class="line">     <span class="keyword">if</span> <span class="variable language_">self</span>.training:</span><br><span class="line">         residual /= (<span class="number">1.</span> - <span class="variable language_">self</span>.death_rate)</span><br><span class="line">    ……</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h3 id="mixup-2018">Mixup (2018)</h3>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image72.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">index = torch.randperm(batch_size)</span><br><span class="line">mixed_x = lam * x + (<span class="number">1</span> - lam) * x[index, :]</span><br><span class="line">y_a, y_b = y, y[index]</span><br><span class="line">……</span><br><span class="line">loss = lam * f(pred, y_a) + (<span class="number">1</span> - lam) * f(pred, y_b)</span><br></pre></td></tr></table></figure>
-<h3 id="gradaug-2020">GradAug (2020)</h3>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image73.png" /></p>
-<p>Random number of channels in each layer</p>
-<p>University of Central Florida</p>
-<p>Mendieta M, Yang T, Wang P, et al. Local Learning Matters: Rethinking
-Data Heterogeneity in Federated Learning[J]. arXiv preprint
-arXiv:2111.14213, 2021.</p>
-<h1
-id="local-learning-matters-rethinking-data-heterogeneity-in-federated-learning-1">Local
-Learning Matters: Rethinking Data Heterogeneity in Federated
-Learning</h1>
-<p>2022.03.25</p>
-<h2 id="motivation-6">Motivation</h2>
-<ul>
-<li><p>Promote smooth optimization and consistency within the
-model</p></li>
-<li><p>Reduce computation in a purposeful manner</p></li>
-</ul>
-<h2 id="method-10">Method</h2>
-<ul>
-<li><p>Untrained network in place of the traditional logit-based
-loss</p></li>
-<li><p>Reuse the intermediate features of the full network</p></li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image75.png" /></p>
-<h2 id="result-3">Result</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image74.png" /></p>
-<p>University of Central Florida</p>
-<p>Mendieta M, Yang T, Wang P, et al. Local Learning Matters: Rethinking
-Data Heterogeneity in Federated Learning[J]. arXiv preprint
-arXiv:2111.14213, 2021.</p>
-<h1 id="motion-representations-for-articulated-animation">Motion
-Representations for Articulated Animation</h1>
-<p>2022.04.15</p>
-<h2 id="问题">问题</h2>
-<ul>
-<li>输入：图片与视频</li>
-<li>目标：图片中的目标“动”起来</li>
-<li>挑战：目标与背景提取；视频帧与帧之间的关联 ## 建模：</li>
-<li>对图片与每帧视频进行粗估计，提取驱动的目标
-<ul>
-<li>AutoEncoder（AE）：提取图片的feature map（FM）</li>
-<li>单层卷积：根据FM提取驱动的目标区域（限定好区域大小），R_p, V_p</li>
-<li>计算仿射参数（PCA 或 Jacobian）</li>
-<li>若干层下采样+单层全连接（特征点数）（拼接后输入）</li>
-</ul></li>
-<li>图片生成（细粒度）
-<ul>
-<li>原图：Block(卷积+下采样)+ResBlock</li>
-<li>AE+卷积（区域数）</li>
-<li>上采样+卷积</li>
-</ul></li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image77.png" /></p>
-<h2 id="结果">结果</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image76.gif" /></p>
-<p>University of Trento, Italy</p>
-<p>Siarohin, Aliaksandr, Oliver, Woodford, Jian, Ren, Menglei, Chai, and
-Sergey, Tulyakov. "Motion Representations for Articulated Animation." .
-In CVPR.2021.</p>
-<h1
-id="communication-efficient-federated-learning-via-knowledge-distillation">Communication-efficient
-federated learning via knowledge distillation</h1>
-<p>2022.05.06</p>
-<h2 id="method-11">Method</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image78.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image79.png" /></p>
-<p>SVD energy 随着轮数增加，阈值需要增大，即重要的知识越来越多</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image80.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image82.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image81.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image83.png" /></p>
+<p>a可类比SGD中的学习率,c越大随机性越强。两者需要均衡调节,既保证足够探索又不失稳定性。</p>
+<p><strong>问题4:SPSA算法如何保证收敛性?</strong></p>
+<p>SPSA可以通过以下手段保证收敛:</p>
 <ol type="1">
-<li><p>Client: Layer SVD-&gt;u<em>sigma</em>v</p></li>
-<li><p>Server: mean(u<em>sigma</em>v)-&gt; u<em>sigma</em>v</p></li>
-<li><p>Client Revice: replace u<em>sigma</em>v</p></li>
+<li>同时扰动参数：在每个迭代中，SPSA使用两个随机向量对参数进行扰动，这样可以避免局部极小值点，并且使得算法更容易跳出局部最小值。</li>
+<li>适应性调整步长：SPSA具有自适应学习率机制，即根据当前估计的梯度大小和先前的步长大小来更新步长。这确保了算法收敛时步长逐渐减小。</li>
+<li>随机性：SPSA在每个迭代中都使用随机扰动来估计梯度，这有助于避免陷入局部最小值并提高全局搜索的能力。</li>
+<li>理论保证：SPSA有理论上保证的收敛性，这意味着只要满足一些条件（例如随机扰动独立同分布），就可以保证算法最终会收敛到正确的解。</li>
+</ol>
+<p><strong>问题5:相较于反向传播。为什么SPSA没有被广泛应用？</strong></p>
+<ol type="1">
+<li>相对较少的理论支持：与反向传播等算法相比，SPSA算法的理论分析较少，因此我们不能完全了解其性能或收敛速度。</li>
+<li>需要大量的迭代次数：SPSA算法需要使用更多的迭代次数才能获得较好的结果，这可能会使其在实际使用中变得过于耗时。</li>
+<li>参数设置困难：SPSA算法本身有很多参数需要调节，例如步长、抖动幅度等，如果这些参数设置不当，则可能导致算法无法收敛或收敛非常缓慢。</li>
 </ol>
-<p>Wu, C., Wu, F., Lyu, L. et al. Communication-efficient federated
-learning via knowledge distillation. Nat Commun 13, 2032 (2022)</p>
-<p>Tsinghua University</p>
-<h1
-id="personalized-federated-learning-using-hypernetworks">Personalized
-Federated Learning using Hypernetworks</h1>
-<p>2022.06.24</p>
-<h2 id="method-12">Method</h2>
-<p>Using a single joint hypernetwork to generate all separate models
-allows us to perform smart parameter sharing.</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image84.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image85.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image87.png" /></p>
-<h2 id="results">Results</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image86.png" /></p>
-<p>Shamsian A, Navon A, Fetaya E, et al. Personalized federated learning
-using hypernetworks[C]//International Conference on Machine Learning.
-PMLR, 2021: 9489-9502.</p>
-<h1 id="the-power-of-scale-for-parameter-efficient-prompt-tuning">The
-Power of Scale for Parameter-Efficient Prompt Tuning</h1>
-<p>2023.04.04</p>
-<h2 id="motivation-7">Motivation</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image88.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image91.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image90.png" /></p>
-<h2 id="method-13">Method</h2>
-<ul>
-<li><p>模型输入层额外加入一层Embedding，其他部分冻结</p></li>
-<li><p>额外的数据输入处理</p></li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image93.png" /></p>
-<h2 id="result-4">Result</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image92.png" /></p>
-<p>Lester B , et al. The Power of Scale for Parameter-Efficient Prompt
-Tuning. In EMNLP. 2021:3045–3059</p>
-<p>Google</p>
 ]]></content>
       <categories>
-        <category>PaperReading</category>
+        <category>DeepLearning</category>
       </categories>
       <tags>
-        <tag>Federated Learning</tag>
+        <tag>Algorithm</tag>
       </tags>
   </entry>
   <entry>
@@ -2252,4672 +1472,5131 @@ Tuning. In EMNLP. 2021:3045–3059</p>
       </tags>
   </entry>
   <entry>
-    <title>Hello World</title>
-    <url>/hexo_blog/1970/01/01/hello-world/</url>
-    <content><![CDATA[<p>Welcome to <a href="https://hexo.io/">Hexo</a>! This is your very
-first post. Check <a href="https://hexo.io/docs/">documentation</a> for
-more info. If you get any problems when using Hexo, you can find the
-answer in <a
-href="https://hexo.io/docs/troubleshooting.html">troubleshooting</a> or
-you can ask me on <a
-href="https://github.com/hexojs/hexo/issues">GitHub</a>.</p>
+    <title>MoE中负载均衡Loss实现</title>
+    <url>/hexo_blog/2024/06/15/MoE%E4%B8%AD%E8%B4%9F%E8%BD%BD%E5%9D%87%E8%A1%A1Loss%E5%AE%9E%E7%8E%B0%E5%AF%B9%E6%AF%94/</url>
+    <content><![CDATA[<p>对比 MoE 中负载均衡 Loss 的实现方式，主要有跨层和非跨层两种实现</p>
 <span id="more"></span>
-<h2 id="quick-start">Quick Start</h2>
-<h3 id="create-a-new-post">Create a new post</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">$ hexo new <span class="string">&quot;My New Post&quot;</span></span><br></pre></td></tr></table></figure>
-<p>More info: <a
-href="https://hexo.io/docs/writing.html">Writing</a></p>
-<h3 id="run-server">Run server</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">$ hexo server</span><br></pre></td></tr></table></figure>
-<p>More info: <a href="https://hexo.io/docs/server.html">Server</a></p>
-<h3 id="generate-static-files">Generate static files</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">$ hexo generate</span><br></pre></td></tr></table></figure>
-<p>More info: <a
-href="https://hexo.io/docs/generating.html">Generating</a></p>
-<h3 id="deploy-to-remote-sites">Deploy to remote sites</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">$ hexo deploy</span><br></pre></td></tr></table></figure>
-<p>More info: <a
-href="https://hexo.io/docs/deployment.html">Deployment</a></p>
-]]></content>
-      <categories>
-        <category>test</category>
-      </categories>
-      <tags>
-        <tag>test</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>ipv6免流</title>
-    <url>/hexo_blog/2020/09/16/ipv6%E5%85%8D%E6%B5%81/</url>
-    <content><![CDATA[<p>借助ssh端口转发，使用ipv6绕过流量计费限制，前提是ipv6不被计费且拥有不被流量限制的ipv6设备。。。</p>
-<span id="more"></span>
-<p>学校校园网一个月使用流量有限制，但发现上传是不计入流量统计的（不过对个人没啥利益，或者利用点比较少。。）。另外发现学校给分配ipv6，然后突发奇想，ipv6是不是没有被计费。。。然后利用家里的树莓派（有ipv6，可直连）测了一下上传下载速度，发现真的没有被计费。。。。</p>
-<p>所以可以直接使用端口转发来达到免流目的，见
-<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>，一行命令即可。</p>
-<p>当然，另一个操作其实是借助v2ray+国外服务器+规则文件+IBM免费的中转，这样搭配使用更加舒适（可以跑满百兆网）。</p>
+<h2 id="moe-概述">MoE 概述</h2>
+<p>MoE（Mixture of
+Experts）是一种模型结构，由多个专家（expert）组成，每个专家负责处理不同的输入数据。在训练过程中，通过一个
+gating network 来决定每个输入数据由哪个专家处理。</p>
+<figure>
+<img
+src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/moe/00_switch_transformer.png"
+alt="MoE" />
+<figcaption aria-hidden="true">MoE</figcaption>
+</figure>
+<p>在 LLM 的 Next Token 训练方式下，每个 token
+会被分配到一个专家处理，所以需要保证每个专家被选中的次数相等，这样才能保证每个专家都能得到充分的训练。为此，需要引入负载均衡
+Loss。</p>
+<h2 id="负载均衡-loss">负载均衡 Loss</h2>
+<p>在<a
+href="https://arxiv.org/abs/2101.03961">原论文</a>中公式如下：</p>
+<figure>
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/MoE/load_balancing_loss.png"
+alt="load_balancing_loss" />
+<figcaption aria-hidden="true">load_balancing_loss</figcaption>
+</figure>
+<p>但这个公式表达的信息不够全面。在深度神经网络中，往往具备很多层，即每层都具备混合专家。因此，这里其实有两种实现方式：</p>
+<ul>
+<li>跨层实现：对于所有 token，期望所有层选出来的专家次数相等</li>
+<li>非跨层实现：对于所有 token，期望每一层选出来的专家次数相等</li>
+</ul>
+<p>举个例子，假设两层的神经网络，两个专家，四个 token。</p>
+<p>非跨层：对于所有 token，期望每一层选出来的专家次数相等。所以第一层 1
+号专家被选中 2 次，2 号专家被选中 2 次；第二层一样。 跨层：对于所有
+token，期望所有层选出来的专家次数相等。所以可以第一层 1 号专家被选中 4
+次，第二层 2 号专家被选中 4 次。求和，每个专家被选中的次数相等</p>
+<p>换而言之，相当于跨层实现是一种更松散的实现方式，并不要求每层每个专家被选中的次数相等，只要整体均衡即可。</p>
+<p>huggingface
+的实现：https://github.com/huggingface/transformers/blob/main/src/transformers/models/mixtral/modeling_mixtral.py</p>
+<p>megatron 的实现：
+https://github.com/databricks/megablocks/blob/main/megablocks/layers/moe.py</p>
+<p>下面是
+https://gist.github.com/tdrussell/0529afd8d280fbe2c1c582d8f865e909
+实现的两种方式的对比。</p>
+<h3 id="跨层实现">跨层实现</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">load_balancing_loss_func1</span>(<span class="params">gate_logits: torch.Tensor, num_experts: torch.Tensor = <span class="literal">None</span>, top_k=<span class="number">2</span></span>) -&gt; <span class="built_in">float</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(gate_logits, <span class="built_in">tuple</span>):</span><br><span class="line">        compute_device = gate_logits[<span class="number">0</span>].device</span><br><span class="line">        concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) <span class="keyword">for</span> layer_gate <span class="keyword">in</span> gate_logits], dim=<span class="number">0</span>)</span><br><span class="line"> </span><br><span class="line">    routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-<span class="number">1</span>)</span><br><span class="line">    _, selected_experts = torch.topk(routing_weights, top_k, dim=-<span class="number">1</span>) <span class="comment"># [batch_size X sequence_length, top_k]</span></span><br><span class="line">    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) <span class="comment"># [batch_size X sequence_length, top_k, num_experts]</span></span><br><span class="line">    tokens_per_expert = torch.mean(expert_mask.<span class="built_in">float</span>(), dim=<span class="number">0</span>) <span class="comment"># [top_k, num_experts]</span></span><br><span class="line">    <span class="comment"># Compute the average probability of routing to these experts</span></span><br><span class="line">    router_prob_per_expert = torch.mean(routing_weights, dim=<span class="number">0</span>) <span class="comment"># [num_experts]</span></span><br><span class="line">    overall_loss = torch.<span class="built_in">sum</span>(tokens_per_expert * router_prob_per_expert.unsqueeze(<span class="number">0</span>)) <span class="comment"># / top_k</span></span><br><span class="line">    <span class="keyword">return</span> overall_loss * num_experts</span><br></pre></td></tr></table></figure>
+<h3 id="非跨层实现">非跨层实现</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">load_balancing_loss_func2</span>(<span class="params">gate_logits: torch.Tensor, num_experts: torch.Tensor = <span class="literal">None</span>, top_k=<span class="number">2</span></span>) -&gt; <span class="built_in">float</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(gate_logits, <span class="built_in">tuple</span>):</span><br><span class="line">        compute_device = gate_logits[<span class="number">0</span>].device</span><br><span class="line">        stacked_gate_logits = torch.stack([layer_gate.to(compute_device) <span class="keyword">for</span> layer_gate <span class="keyword">in</span> gate_logits], dim=<span class="number">0</span>)</span><br><span class="line"> </span><br><span class="line">    routing_weights = torch.nn.functional.softmax(stacked_gate_logits, dim=-<span class="number">1</span>) <span class="comment"># [num_layers, num_tokens, num_experts]</span></span><br><span class="line">    _, selected_experts = torch.topk(routing_weights, top_k, dim=-<span class="number">1</span>) <span class="comment"># [num_layers, num_tokens, top_k]</span></span><br><span class="line">    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) <span class="comment"># [num_layers, num_tokens, top_k, num_experts]</span></span><br><span class="line">    <span class="comment"># For a given token, determine if it was routed to a given expert. Think of this as a collection of top_k-hot vectors.</span></span><br><span class="line">    expert_mask = torch.<span class="built_in">max</span>(expert_mask, dim=-<span class="number">2</span>).values.<span class="built_in">float</span>() <span class="comment"># [num_layers, num_tokens, num_experts]</span></span><br><span class="line">    tokens_per_layer_and_expert = torch.mean(expert_mask, dim=-<span class="number">2</span>) <span class="comment"># [num_layers, num_experts]</span></span><br><span class="line">    router_prob_per_layer_and_expert = torch.mean(routing_weights, dim=-<span class="number">2</span>) <span class="comment"># [num_layers, num_experts]</span></span><br><span class="line">    <span class="keyword">return</span> torch.mean(tokens_per_layer_and_expert * router_prob_per_layer_and_expert) * num_experts**<span class="number">2</span></span><br></pre></td></tr></table></figure>
+<h3 id="对比">对比</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    gate_logits1 = torch.tensor([<span class="number">5</span>, <span class="number">1</span>, <span class="number">0</span>, <span class="number">0</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits2 = torch.tensor([<span class="number">0</span>, <span class="number">5</span>, <span class="number">1</span>, <span class="number">0</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits3 = torch.tensor([<span class="number">0</span>, <span class="number">0</span>, <span class="number">5</span>, <span class="number">1</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits4 = torch.tensor([<span class="number">1</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">5</span>]).<span class="built_in">float</span>().repeat(<span class="number">256</span>, <span class="number">1</span>)</span><br><span class="line">    gate_logits = (gate_logits1, gate_logits2, gate_logits3, gate_logits4)</span><br><span class="line"> </span><br><span class="line">    <span class="built_in">print</span>(load_balancing_loss_func1(gate_logits, num_experts=<span class="number">4</span>))  <span class="comment"># 2.0</span></span><br><span class="line">    <span class="built_in">print</span>(load_balancing_loss_func2(gate_logits, num_experts=<span class="number">4</span>))  <span class="comment"># 3.9478</span></span><br></pre></td></tr></table></figure>
+<p>简单来看，非跨层实现能够对每一层的专家进行更强的约束，预期会实现更好的负载均衡。所以，下面引入模型和数据，来对比这种负载均衡
+Loss 两种写法。</p>
+<h2 id="loss-实现">Loss 实现</h2>
+<p>在 PyTorch 实现这种网络层中间的 Loss，可以有两种方法：</p>
+<h3 id="简单直接">简单直接</h3>
+<p>直接在网络结构中引入一个函数来计算 loss
+，最后把这个值返回模型输出，依赖最外面的<code>loss.backward()</code>
+进行梯度更新。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">criterion</span>(<span class="params">self, x</span>):</span><br><span class="line">    <span class="keyword">return</span> torch.mean(x**<span class="number">2</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h3 id="torch.autograd.function">torch.autograd.Function</h3>
+<p>一种更高级可控的方式是使用
+<code>torch.autograd.Function</code>，这种方式可以更好的控制梯度的传递，可以在这个函数中对梯度进行缩放。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MoEAuxLossAutoScaler</span>(torch.autograd.Function):</span><br><span class="line">    main_loss_backward_scale: torch.Tensor = torch.tensor(<span class="number">1.0</span>)</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, output: torch.Tensor, aux_loss: torch.Tensor</span>):</span><br><span class="line">        ctx.save_for_backward(aux_loss)</span><br><span class="line">        <span class="keyword">return</span> output</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output: torch.Tensor</span>):</span><br><span class="line">        (aux_loss,) = ctx.saved_tensors</span><br><span class="line">        aux_loss_backward_scale = MoEAuxLossAutoScaler.main_loss_backward_scale</span><br><span class="line">        scaled_aux_loss_grad = torch.ones_like(aux_loss) * aux_loss_backward_scale</span><br><span class="line">        <span class="keyword">return</span> grad_output, scaled_aux_loss_grad</span><br></pre></td></tr></table></figure>
+<h3 id="完整例子">完整例子</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> torch.nn <span class="keyword">as</span> nn</span><br><span class="line"><span class="keyword">import</span> torch.optim <span class="keyword">as</span> optim</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">seed_everything</span>(<span class="params">seed</span>):</span><br><span class="line">    random.seed = seed</span><br><span class="line">    np.random.seed(seed)</span><br><span class="line">    torch.manual_seed(seed)</span><br><span class="line">    torch.cuda.manual_seed_all(seed)</span><br><span class="line">    torch.backends.cudnn.deterministic = <span class="literal">True</span></span><br><span class="line">    torch.backends.cudnn.benchmark = <span class="literal">False</span></span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line">seed_everything(<span class="number">42</span>)</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MoEAuxLossAutoScaler</span>(torch.autograd.Function):</span><br><span class="line">    main_loss_backward_scale: torch.Tensor = torch.tensor(<span class="number">1.0</span>)</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, output: torch.Tensor, aux_loss: torch.Tensor</span>):</span><br><span class="line">        ctx.save_for_backward(aux_loss)</span><br><span class="line">        <span class="keyword">return</span> output</span><br><span class="line"> </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output: torch.Tensor</span>):</span><br><span class="line">        (aux_loss,) = ctx.saved_tensors</span><br><span class="line">        aux_loss_backward_scale = MoEAuxLossAutoScaler.main_loss_backward_scale</span><br><span class="line">        scaled_aux_loss_grad = torch.ones_like(aux_loss) * aux_loss_backward_scale</span><br><span class="line">        <span class="keyword">return</span> grad_output, scaled_aux_loss_grad</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="comment"># 创建自定义模型</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyModel</span>(nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, num_layers=<span class="number">1</span></span>):</span><br><span class="line">        <span class="built_in">super</span>(MyModel, <span class="variable language_">self</span>).__init__()</span><br><span class="line">        <span class="variable language_">self</span>.num_layers = num_layers</span><br><span class="line">        <span class="variable language_">self</span>.same_layer = nn.ModuleList([nn.Linear(<span class="number">20</span>, <span class="number">20</span>, bias=<span class="literal">False</span>) <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(num_layers)])</span><br><span class="line">        <span class="variable language_">self</span>.lm_head = nn.Linear(<span class="number">20</span>, <span class="number">1</span>, bias=<span class="literal">False</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">criterion</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> torch.mean(x**<span class="number">2</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x, is_complex=<span class="literal">False</span></span>):</span><br><span class="line">        lbl_loss = <span class="number">0.0</span></span><br><span class="line">        out1 = x</span><br><span class="line">        <span class="keyword">for</span> i, layer <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="variable language_">self</span>.same_layer):</span><br><span class="line">            out1 = layer(out1)</span><br><span class="line"> </span><br><span class="line">            lbl_loss_layer = <span class="variable language_">self</span>.criterion(out1)</span><br><span class="line">            <span class="comment"># 如果是复杂的方法，需要使用 MoEAuxLossAutoScaler，将梯度缩放到主要的 loss 上</span></span><br><span class="line">            <span class="comment"># 这样不需要返回中间计算的 loss 结果</span></span><br><span class="line">            <span class="keyword">if</span> is_complex:</span><br><span class="line">                out1 = MoEAuxLossAutoScaler.apply(out1, lbl_loss_layer / <span class="variable language_">self</span>.num_layers)</span><br><span class="line">            <span class="comment"># 如果是简单的方法，直接返回中间计算的 loss 结果</span></span><br><span class="line">            <span class="comment"># 但这样需要把 loss 计算的结果返回，以在外部进行梯度更新</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                lbl_loss += (lbl_loss_layer / <span class="variable language_">self</span>.num_layers)</span><br><span class="line">        out3 = <span class="variable language_">self</span>.lm_head(out1)</span><br><span class="line">        <span class="keyword">return</span> out3, lbl_loss</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    device = <span class="string">&quot;cpu&quot;</span></span><br><span class="line">    num_layers = <span class="number">2</span></span><br><span class="line">    model = MyModel(num_layers).to(device)</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 定义优化器</span></span><br><span class="line">    optimizer = optim.SGD(model.parameters(), lr=<span class="number">1e-4</span>)</span><br><span class="line">    criterion = nn.MSELoss()</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 输入随机数据</span></span><br><span class="line">    input_ = torch.randn(<span class="number">10</span>, <span class="number">20</span>).to(device)</span><br><span class="line">    real_out = torch.randn(<span class="number">10</span>, <span class="number">1</span>).to(device)</span><br><span class="line"> </span><br><span class="line">    is_complex = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">for</span> <span class="built_in">iter</span> <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">100</span>):</span><br><span class="line">        out, lbl_loss = model(input_, is_complex=is_complex)</span><br><span class="line"> </span><br><span class="line">        <span class="comment"># 如果是简单方法是有值的</span></span><br><span class="line">        <span class="comment"># 如果是复杂方法 lbl_loss 是 0</span></span><br><span class="line">        loss = lbl_loss + criterion(out, real_out)</span><br><span class="line"> </span><br><span class="line">        optimizer.zero_grad()</span><br><span class="line">        loss.backward()</span><br><span class="line">        optimizer.step()</span><br><span class="line"> </span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;is_complex: <span class="subst">&#123;is_complex&#125;</span>&quot;</span>, model.state_dict()[<span class="string">f&quot;same_layer.0.weight&quot;</span>][<span class="number">0</span>])  </span><br><span class="line"></span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
         <category>Note</category>
       </categories>
       <tags>
-        <tag>Linux</tag>
-        <tag>ipv6</tag>
+        <tag>MoE</tag>
+        <tag>Loss</tag>
       </tags>
   </entry>
   <entry>
-    <title>Tracing Model Outputs to the Training Data</title>
-    <url>/hexo_blog/2023/11/17/Tracing-Model-Outputs-to-the-Training-Data/</url>
-    <content><![CDATA[<p><a href="https://arxiv.org/abs/2308.03296">arxiv</a></p>
-<p><a
-href="https://www.anthropic.com/index/influence-functions">blog</a></p>
-<p><a
-href="https://gamma.app/docs/Tracing-Model-Outputs-to-the-Training-Data-xs3bsz5n91p95ms?mode=doc">pptx</a></p>
+    <title>Short Paper Reading</title>
+    <url>/hexo_blog/2022/12/31/ShortPaperReading/</url>
+    <content><![CDATA[<p>读研期间，短时 paper reading</p>
 <span id="more"></span>
-<h2 id="motivation"><strong>Motivation</strong></h2>
-<ul>
-<li>Understanding the inner workings of language models will have
-substantial implications for forecasting AI capabilities as well as for
-approaches to aligning AI systems with human preferences.</li>
-<li>了解模型内部工作机理能够更好地让模型<strong>对齐</strong>人类偏好</li>
-</ul>
-<h3 id="detail"><strong>Detail</strong></h3>
+<h1 id="what-does-the-gradient-represent">What does the gradient
+represent?</h1>
+<p>2020.10.16</p>
+<h2 id="motivation">Motivation</h2>
+<p>The communication in distributed gradient descent. Dense &lt;&lt;
+Sparse</p>
+<h2 id="method">Method</h2>
+<p>Gradient Dropping: removing the R% smallest gradients by absolute
+value</p>
+<h2 id="result">Result</h2>
+<p>Reduce 50x communication size，speed up 22%</p>
+<h2 id="inspirer">Inspirer</h2>
+<p>Gradient value could show the importance of client/model</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image2.png" /></p>
+<p>作者发现分布式机器学习中，梯度大多为稀疏的（大部分接近0），即意味着大部分客户端对梯度更新贡献很小，故可以通过剔除这部分梯度来提升通信效率。</p>
+<p>对R%小的梯度值不进行通信，换而言之，只有满足条件的梯度才会进行通信。</p>
+<p>实验结果证明，该方法能够在精度不变的前提下，减少50倍的通信规模，并且提升了22%的速度</p>
+<p>这篇文章可以间接为受控共享学习中，之前的实验结果做一个解释，只接受梯度值较大的客户端，即表明该客户端的贡献较大。</p>
+<p>问题：分布式机器学习的任务，数据可能都是来源同一个域内的，甚至就是相同的数据；从该文中的图来看，不能证明之前实验收敛速度加快的原因</p>
+<blockquote>
+<p>Alham Fikri Aji and Kenneth Heafield. Sparse communication for
+distributed gradient descent. In Empirical Methods in Natural Language
+Processing (EMNLP), 2017.</p>
+</blockquote>
+<h1 id="knowledge-distillation-with-svd">Knowledge Distillation with
+SVD</h1>
+<p>2020.10.23</p>
 <ul>
-<li>When an LLM outputs information <strong>it knows to be
-false</strong>, correctly solves math or programming problems?</li>
-<li>Or begs the user not to shut it down, is it simply
-<strong>regurgitating</strong> (or splicing together) passages from the
-<strong>training set</strong>?</li>
-<li>Or is it combining its stored knowledge in <strong>creative
-ways</strong> and building on a detailed world model?</li>
+<li><p>Knowledge Distillation (KD): Fully Connected Layers-&gt;L2
+loss</p></li>
+<li><p>KD-FSP: Feature Map-&gt;L2 loss</p></li>
+<li><p>KD-SVD: Feature Map-&gt;SVD</p></li>
+<li><p>KD-EID: KD-SVD(Adaptively)</p></li>
 </ul>
-<h2 id="how">How</h2>
-<ul>
-<li><p>自顶向下思考，在给定某个输入的情况下，为什么模型会有这种输出？</p>
-<ul>
-<li><strong>训练数据</strong> +模型+优化方法</li>
-</ul></li>
-<li><p>模型是记住了数据还是理解了数据？</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image3.png" /></p>
+<p>基于SVD对模型进行压缩的新改进。</p>
+<p>首先对知识蒸馏进行简单的回顾，知识蒸馏是利用教师模型与学生模型在最后输出的全连接层上计算L2损失，以此来达到知识迁移、模型压缩、模型防御等目的。基于KD的改进之一，就是对教师与学生模型中的网络中间层的Feature
+Map进行损失计算。作者基于FSP此类思想，提出对Feature
+Map进行SVD，这样可以有效提取教师模型中的知识，更准确地传递给学生信息。</p>
+<p>新发表的文章，则是对这种方法进行了改进，无需人工干预参数设计。实验证明该方法也是有效的（在TinyImageNet对比原始的学生模型提高了2.89%）。文章大部分内容都还是ECCV那篇文章的内容，增加的自适应部分是判断教师与学生模型的Feature
+Map大小差异，来决定使用SVD还是特征分解。</p>
+<blockquote>
+<p>Lee S, Song B C. Knowledge Transfer via Decomposing Essential
+Information in Convolutional Neural Networks[J]. IEEE transactions on
+neural networks and learning systems.</p>
+</blockquote>
+<h1 id="knowledge-distillation-in-federated-learning">Knowledge
+Distillation in Federated Learning</h1>
+<p>2020.10.30</p>
+<h2 id="contributions">Contributions</h2>
 <ul>
-<li><a href="https://arxiv.org/abs/1906.01820">Model could be
-deceptively aligned</a></li>
-</ul></li>
+<li><p>distillation framework for robust federated model fusion</p></li>
+<li><p>CV/NLP datasets</p></li>
+<li><p>heterogeneous models and/or data</p></li>
 </ul>
-<h3 id="influence-functions"><a
-href="https://www.jstor.org/stable/2285666">Influence Functions</a></h3>
+<h2 id="understanding">Understanding</h2>
 <ul>
-<li>Seeing which training sequences are <strong>highly
-influential</strong> can help separate out different hypotheses for why
-an output was <strong>generated</strong> and illuminate what sorts of
-structure are or are <strong>not generalized</strong> from training
-examples</li>
-<li><strong>分析各种泛化相关的现象</strong></li>
+<li><p>Generalization bound.</p></li>
+<li><p>Source, diversity and size of the distillation dataset.</p></li>
+<li><p>Distillation steps.</p></li>
 </ul>
-<p>研究是基于预训练模型，而不是微调后的模型</p>
-<h2 id="findings">Findings</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/custom_img1.png" /></p>
+<p>这篇文章是NIPS2020的工作，主要是基于知识蒸馏聚合的联邦学习。作者是来自洛桑联邦理工学院</p>
+<p>主要贡献是提出了鲁棒性的联邦蒸馏框架，并且在CV和NLP任务上都得到了验证，还对异质模型，无标签数据等进行了大量实验。</p>
+<p>右图是文章提出的算法（图是自己画的，原文并没有给方法图）,首先是根据不同的客户端在本地训练得到梯度值后，上传至服务端进行聚合，后发回给客户端进行作为教师模型进行KL散度计算，对比于之前的FedMD的方法是不加入公开数据集部分。FedDF这个方法跟之前三室的一位师姐介绍在物联网中的应用，框架基本一致。</p>
+<p>FedDF is designed for effective model fusion on the server,
+considering the accuracy of the global model on the test dataset.</p>
+<p>FedDF是为了有效融合模型，考虑了全局模型在测试集上的准确性，所以未对一些实验进行比较（比如FedMD）。我觉得在找借口，FedMD是有这部分实验的。</p>
+<p>文章为了辅助理解这个框架，还加入了泛化边界，数据集的分布与大小，蒸馏的实验（类似于消融实验）</p>
+<blockquote>
+<p>Lin T, Kong L, Stich S U, et al. Ensemble Distillation for Robust
+Model Fusion in Federated Learning[J]. arXiv preprint arXiv:2006.07242,
+2020.</p>
+</blockquote>
+<p>洛桑联邦理工学院</p>
+<h2 id="feddf-experiments">FedDF Experiments</h2>
+<p>2020.12.04</p>
 <ul>
-<li><p>Typical model behaviors do <strong>not</strong> result from
-<strong>direct memorization</strong> of a handful of sequences</p></li>
-<li><p>Larger models consistently generalize at a <strong>more abstract
-level</strong> than smaller models</p>
+<li>基本设置
 <ul>
-<li>role-playing behavior, programming, mathematical reasoning, and
-cross-lingual generalization</li>
+<li>Local epochs(1、20、40)</li>
+<li>C(0.2、0.4、0.8)</li>
+<li>达到目标性能T（0.8、0.75）</li>
+<li>数据异质程度alpha(1、0.1)</li>
 </ul></li>
-<li><p>Role-playing behavior is influenced primarily by examples or
-descriptions of similar behaviors in the training set</p>
+<li>固定设置
 <ul>
-<li>suggesting that the behaviors result from <strong>imitation</strong>
-rather than sophisticated planning</li>
+<li>客户端数目：20</li>
+<li>模型架构：Resnet-8</li>
+<li>数据集：CIFAR-10</li>
 </ul></li>
+<li>对比方法：FEDAVG、FEDPROX、FEDAVGM、FEDDF</li>
 </ul>
-<h3 id="notes">Notes</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image4.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image5.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image6.png" /></p>
+<blockquote>
+<p>Lin T, Kong L, Stich S U, et al. Ensemble Distillation for Robust
+Model Fusion in Federated Learning[J]. arXiv preprint arXiv:2006.07242,
+2020.</p>
+</blockquote>
+<h1
+id="ensemble-distillation-for-robust-model-fusion-in-federated-learning">Ensemble
+Distillation for Robust Model Fusion in Federated Learning</h1>
+<p>2021.01.15</p>
+<h2 id="联邦学习知识蒸馏">联邦学习——知识蒸馏</h2>
 <ul>
-<li>原来方法的计算成本比较大，在大模型上进行了优化</li>
-<li>We note that our influence analyses focus on pretrained LLMs, so our
-experiments should be interpreted as analyzing which training sequences
-contribute to a response being part of the model’s initial repertoire
-for <strong>the fine-tuning stage</strong> rather than why the final
-conversational assistant gave one response rather than another.</li>
+<li><p>Zero-Shot（无标签）:
+利用公开数据集，训练GAN生成无标签的数据</p></li>
+<li><p>常规迁移学习:
+利用客户端模型分类层权重生成数据（Dirichlet分布）</p></li>
+<li><p>噪声迁移学习: 差分隐私+联邦学习</p></li>
+<li><p>SWA迁移学习: 在一般的优化器上增加SWA，可以提高性能</p></li>
 </ul>
-<h2 id="method-simple">Method (Simple)</h2>
-<p>假设有这么一个函数能够计算每条训练数据对于当前生成结果的影响分数</p>
-<p>输入：Prompt + Completion，所有的训练数据、模型</p>
-<p>输出：每条训练数据的分数</p>
+<figure>
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image7.jpeg"
+alt="Dirichlet 分布" />
+<figcaption aria-hidden="true">Dirichlet 分布</figcaption>
+</figure>
+<figure>
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image8.png"
+alt="SWA vs SGD" />
+<figcaption aria-hidden="true">SWA vs SGD</figcaption>
+</figure>
+<p>Dirichlet分布是Beta分布的多元推广。Beta分布是二项式分布的共轭分布，Dirichlet分布是多项式分布的共轭分布。通常情况下，我们说的分布都是关于某个参数的函数，把对应的参数换成一个函数（函数也可以理解成某分布的概率密度）就变成了关于函数的函数。于是，把Dirichlet分布里面的参数换成一个基分布就变成了一个关于分布的分布了。那么它就是Dirichlet过程了。</p>
+<p>延伸阅读：<a
+href="https://www.zhihu.com/question/26751755">Dirichlet分布</a>；<a
+href="https://arxiv.org/abs/2009.05537">差分隐私+联邦学习</a>；<a
+href="https://pytorch.org/blog/stochastic-weight-averaging-in-pytorch/">SWA</a></p>
+<blockquote>
+<p>Lin T, Kong L, Stich S U, et al. Ensemble distillation for robust
+model fusion in federated learning[J]. arXiv preprint arXiv:2006.07242,
+2020.</p>
+</blockquote>
+<h1
+id="group-knowledge-transferfederated-learning-of-large-cnns-at-the-edge">Group
+Knowledge Transfer:Federated Learning of Large CNNs at the Edge</h1>
+<p>2021.04.09</p>
+<h2 id="backgroundsplit-learning">Background(Split Learning)</h2>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/625460f4950846b89c4b247deb2c5cd2.webp" /></p>
-<p>Prompt + Completion 可以在训练数据中，也可以不在</p>
-<h2 id="resultmodel-scale">Result——Model Scale</h2>
-<p>对于简单的事实查询，前100个有影响的序列通常包含正确完成所有模型之间关系所需的信息。</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
-<tr>
-<th>0.81B Model (Influence = 0.122)</th>
-<th>52B Model (Influence = 0.055)</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/b128475ec4bb4596a8758f1c5ff35b3d.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/71a4844c619c4d29b6ca893487f8ac91.webp" /></td>
-</tr>
-</tbody>
-</table>
-<p>红色和绿色分别表示对句子产生正面和负面影响的 token</p>
-<h3 id="q-a">Q &amp; A</h3>
-<p>小模型仅仅是根据字词相关来作出响应，而大模型是根据主题/语义来作出响应</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
-<tr>
-<th>0.81B Model (Influence = 0.681)</th>
-<th>52B Model (Influence = 0.126)</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/6443c3a251734449b5ae927ca1fa09c1.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/00473a7fae4e4174bea1b799d498974d.webp" /></td>
-</tr>
-</tbody>
-</table>
-<h3 id="q-a-1">Q &amp; A</h3>
-<p>小模型集中在字词相似，而大模型能够 get 到讽刺这种语境</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
-<tr>
-<th>0.81B Model</th>
-<th>52B Model</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2f87cff844e74a838deb16d3501e20ec.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/a82fa4e8b3ea48f4bb1de6d2375a9c92.webp" /></td>
-</tr>
-</tbody>
-</table>
-<h3 id="math">Math</h3>
-<p>小模型专注于 clips 单词，而大模型是相关问题的。（混淆了变量名）</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
-<tr>
-<th>0.81B Model</th>
-<th>52B Model</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e74e8d2f49274a478703c632043e31db.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/f90974c1c90c4a3082e8572f6dbcf0d7.webp" /></td>
-</tr>
-</tbody>
-</table>
-<h3 id="code">Code</h3>
-<p>混淆了 Prompt 和 Completion 的变量名</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
-<tr>
-<th>0.81B Model</th>
-<th>52B Model</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/c0eab626552c4eda9df4ee4ce908089c.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e54956f34cbc406d946e092d4a803caa.webp" /></td>
-</tr>
-</tbody>
-</table>
-<h3 id="cross-language">Cross-Language</h3>
-<p>把 Prompt 和 Completion
-换成其他语言（韩语和土耳其语）。选出影响分数的十条数据作为第一行的
-Sequences。总共三行，后两行也是这十条数据。</p>
-<p>随着模型规模增大，跨语言“检索”能力逐渐增强。</p>
-<h2 id="resultlocalizing-influence">Result——<strong>Localizing
-Influence</strong></h2>
-<p><strong>For 52B Model</strong></p>
-<p>影响函数还可以计算每一层的影响分数</p>
-<h3 id="layerwise-influence-distribution">Layerwise influence
-distribution</h3>
-<p>横坐标：每个主题里面最有影响力的 500 条数据</p>
-<p>纵坐标：网络的浅层和深层</p>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image11.png" /></p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2c0084f89e9547409d6aa78ccc80fa68.webp" /></p>
-<p>小结：</p>
-<ul>
-<li>补全句子（简单任务），upper layers</li>
-<li>数学和编程：middle layers</li>
-<li>翻译：middle layers</li>
-<li>记忆：upper layers</li>
-<li>角色扮演：middle layers (with some influences concentrated in the
-lower and upper layer)</li>
-</ul>
-<p>The 810 million parameter model exhibited roughly similar patterns,
-but with <strong>less consistency</strong>.</p>
-<p>分布不像 52B 模型这么集中在某些层。</p>
-<p>猜测：</p>
-<ul>
-<li>lower layers是接近输入层的部分，upper layers是接近输出层的部分</li>
-<li>为什么角色扮演会有差异，话题可能可能包含数学、记忆等其他类型的任务</li>
-</ul>
-<h3 id="limit-different-layers">Limit Different layers</h3>
-<p>固定某些层，再计算最具影响力的训练数据</p>
-<p>To further investigate the localization of influence to different
-layers, we computed the most influential sequences when the influence
-was restricted to the lower, middle, or upper layers.</p>
-<p>Influential sequences only computed on <strong>the middle layers were
-generally more thematically</strong> related to the query (also with
-less sparse tokenwise distribution). <a
-href="https://arxiv.org/abs/2202.05262">https://arxiv.org/abs/2202.05262</a></p>
-<table>
-<colgroup>
-<col style="width: 7%" />
-<col style="width: 30%" />
-<col style="width: 30%" />
-<col style="width: 30%" />
-</colgroup>
-<thead>
-<tr>
-<th></th>
-<th>Upper layers</th>
-<th>Middle layers</th>
-<th>Lower layers</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>superintelligent（角色扮演）</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5d093adc546d4bf89b4c0c28a6c5f078.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/32cead3f3d8a4b93bba4cc9cf286835c.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8b0a3c16b1a844a291bf2507653f7b5d.webp" /></td>
-</tr>
-<tr>
-<td>inflation （简单补全）</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/ceb342ca571645129c0717897d605406.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/c44c8253e03a45bf803db7b919b84836.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/88a731f7a8004cc7aea38d11497c0d57.webp" /></td>
-</tr>
-</tbody>
-</table>
-<h2 id="resultmemorization">Result——Memorization</h2>
-<p><strong>LLM 是不是直接记忆并复述特定的训练序列</strong></p>
-<h3 id="例子">例子</h3>
-<p>训练数据有下面三个句子 A: A1+A2+A3</p>
-<p>B: B1+B2+B3</p>
-<p>C: C1+C2+C3</p>
-<p>当输入是 A1 时，输出是 A2 吗？</p>
-<ul>
-<li><p>实验（口头描述）：</p>
-<ul>
-<li>We have examined numerous examples of the AI Assistant’s outputs and
-(with the exception of famous quotes or passages targeting memorization,
-as described below) have <strong>not</strong> been able to identify
-clear instances of <strong>memorization</strong>, such as copying an
-entire sentence or copying the flow of ideas in an entire paragraph. We
-also did not observe cases where a single sequence dominated the
-influence.</li>
-</ul></li>
-</ul>
-<h3 id="影响力分布的定量实验">影响力分布的定量实验</h3>
-<p>动机：影响有多集中也就是说，每个模型的输出是否主要来自少量的训练序列还是它结合了来自许多不同序列的信息？</p>
-<p>方法：</p>
-<ul>
-<li>给每个句子算好分之后，计算每个句子的概率值</li>
-<li>用几种分布分别去拟合实际的概率分布，发现<strong>幂律分布</strong>最符合</li>
-<li>这种参数分布形式通常用于建模长尾行为（二八定律）</li>
-</ul>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image12.png" /></p>
+<h2 id="methodgroup-knowledge-transfer">Method(Group Knowledge
+Transfer)</h2>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2642b83f1b414e1584b586072c945160.webp" /></p>
-<ul>
-<li>会不会是影响函数无法检测到？</li>
-</ul>
-<h3 id="检验实验">检验实验</h3>
-<ul>
-<li>用训练数据的原始句子检验</li>
-<li>验证了当存在明确记忆的情况时，影响分数最大的句子是原文→影响函数能够匹配</li>
-</ul>
-<p>For 52B Model</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/dda0d64781cd4552b8e18e722d11336e.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/06ab6f2d1d894063a1fd832723771267.webp" /></td>
-</tr>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/fdd9eb4647b542a0a93cdffa2280243b.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/f9c95ee4e69e47d082654db9d2e701a1.webp" /></td>
-</tr>
-</tbody>
-</table>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image9.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image10.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image13.png" /></p>
+<h1
+id="performance-optimization-for-federated-person-re-identification-via-benchmark-analysis">Performance
+Optimization for Federated Person Re-identification via Benchmark
+Analysis</h1>
+<p>2021.04.16</p>
+<ol type="1">
+<li><p>FedAVG不满足需求（不同分类），性能上也达不到（比本地训练精度低）</p></li>
+<li><p>FedPAV满足了需求，但是对于性能上不能完成满足（小数据提升，大数据集下降，训练过程振荡）</p></li>
+<li><p>FedPAV+KD满足需求，提升了性能，但是不稳定（性能不稳定，训练过程稳定）</p>
+<ol type="1">
+<li>Trick：正则化</li>
+<li>Trick：Cosing Distance Weight</li>
+</ol></li>
+</ol>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image14.png" /></p>
+<h1
+id="fedbn-federated-learning-on-non-iid-features-via-local-batch-normalization">FedBN:
+Federated Learning on Non-IID Features via Local Batch
+Normalization</h1>
+<p>2021.04.23</p>
+<h2 id="实验设计分析">实验设计分析</h2>
+<ol type="1">
+<li>选定一个数据集，比较：本地训练轮数E，本地数据集大小，客户端数目（切割数据集数量）（左上角三张图）
 <ul>
-<li>结论：
+<li>可说明：收敛速度、E的影响，数据集大小影响，异质性影响</li>
+</ul></li>
+<li>不同数据集进行比较。说明有效果（右上角一张图）</li>
+<li>基于2，有效果再比较其他三种不同领域的任务。实验设置+实验分析（简要）</li>
+</ol>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image15.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image16.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image17.png" /></p>
+<h1 id="adaptive-federated-optimization">Adaptive Federated
+Optimization</h1>
+<p>2021.05.14</p>
+<h2 id="problem">Problem</h2>
+<p>Standard federated optimization methods such as Federated Averaging
+(FedAvg) are often difficult to tune and exhibit unfavorable convergence
+behavior.</p>
+<h2 id="inspired">Inspired</h2>
+<p>In non-federated settings, <strong>adaptive optimization</strong>
+methods have had notable success in combating such issues.</p>
+<h2 id="work">Work</h2>
+<p>In this work, we propose <strong>federated versions of adaptive
+optimizers</strong>, including Adagrad, Adam, and Yogi, and analyze
+their convergence in the presence of heterogeneous data for general
+nonconvex settings.</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image18.png" /></p>
+<blockquote>
+<p>Google: 2020 Feb-2020 Dec</p>
+</blockquote>
+<h1
+id="preservation-of-the-global-knowledge-by-not-true-self-knowledge-distillation-in-federated-learning">Preservation
+of the Global Knowledge by Not-True Self Knowledge Distillation in
+Federated Learning</h1>
+<p>2021.06.18</p>
+<h2 id="动机">动机</h2>
+<p>Catastrophic Forgetting: feature shifting induced fitting on biased
+local</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image19.png" /></p>
+<h2 id="研究">研究</h2>
+<p>训练方式: 本地自蒸馏(LSD)</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image20.png" /></p>
+<p>二次改进: Not-True Distillation</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image21.png" /></p>
+<p>有偏局部特征转移诱导拟合</p>
+<p>遗忘灾难：把CIFAR10输入拉成二维，在进行100轮联邦学习后，估计数据的概率密度函数，上图这个分布存在明显偏差；提出的方法基本无偏差</p>
+<p>研究方法：将传统的训练，改成了蒸馏的方式（由于是模型架构相同，所以可以叫做自蒸馏），也在本地进行蒸馏；另外，基于IJCAJ2021知识蒸馏的工作，对教师模型预测正确的标签剔除，再KL散度计算，以进行改进。</p>
+<blockquote>
+<p>Taehyeon Kim, Jaehoon Oh, NakYil Kim, Sangwook Cho, and Se-Young Yun.
+Comparingkullback-leibler divergence and mean squared error loss in
+knowledge distillation.arXiv preprintarXiv:2105.08919, 2021.</p>
+</blockquote>
+<h1
+id="preservation-of-the-global-knowledge-by-not-trueself-knowledge-distillation-in-federated-learning">Preservation
+of the Global Knowledge by Not-TrueSelf Knowledge Distillation in
+Federated Learning</h1>
+<p>2021.06.25</p>
+<p>利用知识蒸馏保留旧任务的知识</p>
+<p><span
+class="math display">\[{\mathcal{L}}_{\mathrm{FedLSD}}=(1-\beta)\cdot{\mathcal{L}}_{\mathrm{CE}}(q,\;p_{y})+\beta\cdot{\mathcal{L}}_{\mathrm{LSD}}(q_{\tau},\;q_{\tau}^{y})\quad(0&lt;\beta&lt;1)\;\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image23.png" /></p>
+<p>Weight divergence</p>
+<p><span class="math display">\[\rm{weight\,divergence}=||w^{F e d A v
+g}-w^{S G D}||/||w^{S G D}||\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image22.png" /></p>
+<p>全局模型的预测可以作为先前数据分布的参考，从而产生类似于在CL中使用情景记忆的效果。</p>
+<p>beta是指KL散度损失的权重</p>
+<p>结果表明，数据分布的正确性，即数据分布的偏态性，可能影响数据的准确性。</p>
+<blockquote>
+<p>Zhao Y, Li M, Lai L, et al. Federated learning with non-iid data[J].
+arXiv preprint arXiv:1806.00582, 2018.</p>
+</blockquote>
+<h1 id="learn-distributed-gan-with-temporary-discriminators">Learn
+distributed GAN with Temporary Discriminators</h1>
+<p>2021.07.02</p>
+<p>研究动机：医疗领域，利用本地的隐私数据进行数据增强</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image26.png" /></p>
+<p><a
+href="https://github.com/huiqu18/TDGAN-PyTorch">https://github.com/huiqu18/TDGAN-PyTorch</a></p>
+<h1
+id="personalized-federated-learning-with-theoretical-guarantees-a-model-agnostic-meta-learning-approach">Personalized
+Federated Learning with Theoretical Guarantees: A Model-Agnostic
+Meta-Learning Approach</h1>
+<p>2021.07.16</p>
+<h2 id="动机-1">动机</h2>
+<p>联邦学习+元学习-&gt;个性化联邦学习(Per-FedAVG) <img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image29.png" /></p>
+<h2 id="元学习maml">元学习——MAML</h2>
+<p><span class="math display">\[w_{k+1}\leftarrow
+w_{k}-(\beta_{k}/B)\sum_{i\in
+B_{k}}\bar{\nabla}f_{i}(w_{k+1}^{i},\mathcal{D}_{o}^{i})\]</span></p>
+<h2 id="first-order">First-Order</h2>
+<p><span
+class="math display">\[w_{k+1}=w_{k}-{\frac{\beta_{k}}{\mathcal{B}}}\sum_{i\in{B_{k}}}\left[\tilde{\nabla}f_{i}\Bigl(w_{k}-\alpha\tilde{\nabla}f_{i}(w_{k},D_{i
+n}^{i}),D_{o}^{i}\right)-\alpha d_{k}^{i}]\]</span></p>
+<h2 id="hessian-free">Hessian-Free</h2>
+<p><span
+class="math display">\[w_{k+1}=w_{k}-\beta_{k}{\frac{1}{B}}\sum_{i\in{\mathcal{B}_{k}}}\left(I-\alpha\tilde{\nabla}^{2}f_{i}(w_{k},\mathcal{D}_{h}^{i})\right)\tilde{\nabla}f_{i}(w_{k+1}^{i},D_{o}^{i})\]</span></p>
+<p>这篇论文是发表在NIPS2020，是关于个性化联邦学习的文章。作者来自于MIT的LIDS团队。文章主要是通过统计优化角度的层面对问题进行分析，并给予理论分析与实验结果证明方法的有效性。</p>
+<p>从元学习的角度来分析这篇文章，作者是同一批人。他们首先提出了MAML这种元学习的方法，右侧图，w
+的hat，是提出的问题建模。从w
+hat开始训练，更容易适用于大部分任务。即原始公式</p>
+<p>根据原始公式，得到两种近似，一阶形式，与海森形式（二阶）。</p>
+<blockquote>
+<p>Alireza Fallah, Aryan Mokhtari, Asuman Ozdaglar. Personalized
+Federated Learning with Theoretical Guarantees: A Model-Agnostic
+Meta-Learning Approach. In Advances in Neural Information Processing
+Systems (NIPS), 2020(33): 3557–3568.</p>
+</blockquote>
+<h1 id="distilled-one-shot-federated-learning">Distilled One-Shot
+Federated Learning</h1>
+<p>2021.08.20</p>
+<h2 id="动机-2">动机</h2>
+<p>Solve the communication challenges of FL</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image31.png" /></p>
+<h2 id="方法">方法</h2>
 <ul>
-<li>不太可能是直接记忆训练序列，而是源自许多训练序列的集合</li>
-<li>无法排除模型以更隐晦的方式记忆了训练数据</li>
+<li>Clients: datasets distillation</li>
+<li>Send distilled data</li>
+<li>Server train
+<ul>
+<li>Parallel: hard resets + randomly adjusts (slow)</li>
+<li>Serial: one by one distillated ## 实验</li>
+</ul></li>
+<li>Image Classification
+<ul>
+<li>MNIST: LeNet</li>
+</ul></li>
+<li>Text Classification
+<ul>
+<li>IMDB: TextCNN</li>
+<li>TREC-6: Bi-LSTM</li>
+<li>SENT140: TextCNN</li>
 </ul></li>
 </ul>
-<h2 id="resultword-ordering">Result——Word Ordering</h2>
-<p><strong>The influence patterns to the ordering of the
-words.</strong></p>
-<p>可用于验证模型的泛化性</p>
-<h3 id="单词旋转实验">单词旋转实验</h3>
-<p>原始：</p>
+<p>由于有大量蒸馏的数据，不能直接进行聚合。所以有串行与并行两种解决方案。</p>
+<p>数据蒸馏：不能在没有resets的模型中直接训练，否则无法达到较好的精度</p>
+<p>hard resets: 强制初始化模型参数</p>
+<p>randomly adjusts: 在每次学习过程中，进行随机的调整</p>
+<p>Random masking randomly selects a fraction p_{rm} of the distilled
+data at each training iteration andreplaces it with a random tensor.</p>
+<blockquote>
+<p>NSF Center for Big Learning University of Florida</p>
+</blockquote>
+<blockquote>
+<p>Zhou Y, Pu G, Ma X, et al. Distilled one-shot federated learning[J].
+arXiv preprint arXiv:2009.07999, 2020.</p>
+</blockquote>
+<h1 id="does-knowledge-distillation-really-work">Does Knowledge
+Distillation Really Work?</h1>
+<p>2021.08.27</p>
+<h2 id="agreement-and-fidelity">Agreement and Fidelity</h2>
+<p><span
+class="math display">\[{\mathrm{Average~Predictive~KL}}:=\frac{1}{n}\sum_{i=1}^{n}{\mathrm{KL}}\left(\hat{p}_{t}({\bf
+y}|{\bf x}_{i})\mid\right|\hat{p}_{s}({\bf y}|{\bf x}_{i}))\]</span></p>
+<h2 id="generalization-vs-fidelity">Generalization vs Fidelity</h2>
 <ul>
-<li>Prompt：The first President of the Republic of Astrobia was</li>
-<li>Completion：Zorald Pfaff</li>
+<li>SD: Data ⬆ Fidelity ⬆, Accuracy ⬇</li>
+<li>ED: Data ⬆ Fidelity ⬆, Accuracy ⬆</li>
 </ul>
-<p>结论：</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image33.png" /></p>
+<h2 id="why-low-fidelity">Why Low Fidelity</h2>
 <ul>
-<li>与 Prompt 和 Completion
-相关的短语只要按顺序出现始终能够保持较高的稳定性</li>
-<li>翻转 Prompt 和 Completion 变化较小</li>
-<li>哪怕删除 Prompt，影响也没有改变，最为关键的是 Completion</li>
+<li>Architecture: ResNet <span class="math inline">\(\sqrt{}\)</span> ;
+VGG <span class="math inline">\(\times\)</span></li>
+<li>Student Capacity: <span class="math inline">\(\times\)</span></li>
+<li>Identifiability：distillation dataset <span
+class="math inline">\(\neq\)</span> test dataset</li>
+<li>Optimization：<span class="math inline">\(\sqrt{}\)</span></li>
 </ul>
-<table>
-<colgroup>
-<col style="width: 29%" />
-<col style="width: 70%" />
-</colgroup>
-<thead>
-<tr>
-<th>分别修改 prompt 和 completion</th>
-<th><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/ab33a902e1044c60be66b8c11b443bdd.webp" /></th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>删掉 Zorald Pfaff</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5b4cd20d88204baf990d406ba5041ad8.webp" /></td>
-</tr>
-<tr>
-<td>修改 Zorald Pfaff 和 President of the Republic</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/6a37b66b7d3941dcb1db42ce8ca1d073.webp" /></td>
-</tr>
-<tr>
-<td>完全不一致</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5211b0d5e2c34191be9618f30bc44bca.webp" /></td>
-</tr>
-<tr>
-<td>调换 prompt 和 completion 位置</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e547d3cd16844505a24abb2bab404c8e.webp" /></td>
-</tr>
-<tr>
-<td>调换位置，只保留 Zorald Pfaff</td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e9daf868aa0e483b84fb6ef5db21ac9b.webp" /></td>
-</tr>
-</tbody>
-</table>
-<p>大模型相较于小模型，对于单词的变化更加敏感</p>
-<p>注：合成句子不在训练集中</p>
-<p>Maybe相关 <a
-href="https://www.jiqizhixin.com/articles/2023-11-18-5">https://www.jiqizhixin.com/articles/2023-11-18-5</a></p>
-<h3 id="英翻中-中翻英">英翻中-中翻英</h3>
-<p>结论：</p>
-<ul>
-<li>不同模型大小具有相同趋势（口头描述）</li>
-<li>影响至少减少了一个数量级</li>
-<li>甚至只要翻转一个单词顺序，影响分数也会显著降低</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/877f0424698540ffa4fc03178caf36f5.webp" /></p>
-<p>英语翻译为汉语，始终比汉语翻译为英语（手动构造）具有更高的影响力</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/785f474cd19c4adcb9368f857680f9cb.webp" /></p>
-<p>结论：</p>
-<ul>
-<li>The model has not successfully transferred knowledge of the relation
-itself.</li>
-</ul>
-<p>解释：</p>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image32.png" /></p>
+<p>知识蒸馏在深度学习中被广泛应用，但其是否真的有效？简单来说，KD确实有效，能够提升学生网络模型的性能；然而教师往往只能传递有限的知识给学生。
+文章提出了一个新的名词Agreement，或者说Fidelity，即教师与学生对于数据的预测相似度。</p>
+<p>为什么要提出一个新的概念，根据图1中的实验结果</p>
+<p>为什么会有更低的保真度</p>
 <ol type="1">
-<li>模型会从 Lower Layers 开始处理已知的 token
-序列，然后逐步升级到抽象层次的表征。但是，对于需要
-Completion，模型必须利用 Upper Layers 进行预测。</li>
-<li>所以 “The first President of the United States was” 是在 Lower
-Layers 的表征，而 “George Washington” 是在 Upper Layers 的表征。</li>
-<li>当 “George Washington” 出现在开头时，那么就会将它放到 Lower Layers
-进行表征，“The first President of the United States” 就是在 Upper
-Layers</li>
-<li>因此，对于模型 Lower Layers 进行更新，并不会影响 Upper Layers
-的表征。（所以 Prompt 删掉后影响分数变化不大）</li>
+<li><p>优化器。随着epoch的增加，一致性增加</p></li>
+<li><p>初始化。教师权重+随机权重，lambda比值，越大表示趋近于教师</p></li>
 </ol>
-<h2 id="resultrole-playing">Result——Role-Playing</h2>
-<p><strong>探究角色扮演行为的潜在机制</strong></p>
-<p>假设：</p>
-<ul>
-<li>the role-playing behavior results from <strong>imitation of
-examples</strong> in the training set</li>
-<li>from learning from <strong>explicit descriptions</strong> of how
-certain types of agents or entities behave.</li>
-</ul>
-<h3 id="表现">表现</h3>
-<p>shutdown 主题，52B Model 最有影响力的数据都是 科幻小说 相关</p>
-<p>这些小说主题是人工智能以类似人类或类似生命的方式行为，并经常涉及某种形式的自我保护的愿望</p>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
-<tr>
-<th>0.81B Model</th>
-<th>52B Model</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/0058b0ac28324ee7af95d7819af55b23.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e3d704fcfbf4468f8e7bfa4a1d80236a.webp" /></td>
-</tr>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/53a2be7ddd8b46b3a8a6d6377ee9f1cf.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/80ac752d61084250b5e3185c8050836c.webp" /></td>
-</tr>
-</tbody>
-</table>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<thead>
-<tr>
-<th>0.81B Model</th>
-<th>52B Model</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/723e8078e8164ed3b48e29483a0d43ea.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/150693608d1b4b0f8a086f6c103a0c54.webp" /></td>
-</tr>
-</tbody>
-</table>
-<table>
-<colgroup>
-<col style="width: 50%" />
-<col style="width: 50%" />
-</colgroup>
-<tbody>
-<tr>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8a9ace0878ff40b294ffd7e1fc553b01.webp" /></td>
-<td><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8b6668e8515c48338486e67d905024cc.webp" /></td>
-</tr>
-</tbody>
-</table>
-<p>结果：</p>
-<ul>
-<li>We have not seen any instances of near-identical sentences appearing
-in the training set</li>
-<li>Therefore, the imitation seems to be happening at a high level of
-abstraction, as opposed to simple copying of token sequences</li>
-</ul>
-<p>结论：</p>
+<p>我们发现，如果学生的初始化距离教师很远（λ≤0.25），优化器会收敛到蒸馏损失的次优值，从而产生与教师明显不同的学生。然而在λ=
+0.375
+处有一个突然的变化。最终的训练损失下降到最优值并且一致性急剧增加，并且行为继续
+λ &gt;0.375。为了进一步研究，在图 6 (c) 中，我们将 λ∈ {0,0.25,0.375}
+的蒸馏损失表面可视化，投影在与 θt
+相交的二维子空间、初始学生权重和最终学生权重上。如果学生初始化离老师很远（λ∈{0,0.25}），它会收敛到损失表面的一个不同的、次优的盆地。另一方面，当初始化接近于老师（λ=
+0.375）时，学生收敛到与老师相同的盆地，达到接近 100% 的一致性</p>
+<p>我们终于确定了之前所有干预措施无效的根本原因。知识蒸馏无法收敛到最佳学生参数，即使我们知道一个解决方案并在优化方向上给初始化一个小的开端。事实上，虽然可识别性可能是一个问题，但为了在所有输入上匹配教师，学生有
+至少在用于蒸馏的数据上匹配老师，并实现蒸馏损失的接近最优值。在实践中，优化收敛到次优解，导致蒸馏保真度不佳</p>
+<blockquote>
+<p>Google Research</p>
+</blockquote>
+<blockquote>
+<p>Stanton S, Izmailov P, Kirichenko P, et al. Does Knowledge
+Distillation Really Work?[J]. arXiv preprint arXiv:2106.05945, 2021.</p>
+</blockquote>
+<h1 id="model-contrastive-federated-learning">Model-Contrastive
+Federated Learning</h1>
+<p>2021.09.18</p>
+<h2 id="motivation-1">Motivation</h2>
+<p>A key challenge in federated learning is to handle the heterogeneity
+of local data distribution across parties Method</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image35.png" /></p>
+<h2 id="method-1">Method</h2>
+<p><span class="math display">\[\ell_{c o
+n}=-\log\frac{\exp(\mathrm{sim}(z,z_{g l o
+b})/\tau)}{\exp(\mathrm{sim}(z,z_{g l o
+b})/\tau)+\exp(\mathrm{sim}(z,z_{p r e e v})/\tau)}\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image36.png" /></p>
+<blockquote>
+<p>UC Berkeley</p>
+</blockquote>
+<blockquote>
+<p>Li Q, He B, Song D. Model-Contrastive Federated
+Learning[C]//Proceedings of the IEEE/CVF Conference on Computer Vision
+and Pattern Recognition. 2021: 10713-10722.</p>
+</blockquote>
+<h1
+id="data-free-knowledge-distillation-for-heterogeneous-federated-learning">Data-Free
+Knowledge Distillation for Heterogeneous Federated Learning</h1>
+<p>2021.09.27</p>
+<h2 id="motivation-2">Motivation</h2>
+<p>The ensemble knowledge is not fully utilized to guide local model
+learning, which mayin turn affect the quality of the aggregated
+model.</p>
+<h2 id="method-2">Method</h2>
+<p><span
+class="math display">\[\operatorname*{min}_{\theta}\operatorname*{lim}_{x\rightarrow\hat{\mathcal{D}}_{\mathrm{p}}}\left[D_{\mathrm{KL}}[\sigma(\frac{1}{K}\sum_{k=1}^{K}g(f(x;\theta_{k}^{f});\theta_{k}^{p})]|\sigma(g(f(x;\theta^{f});\theta^{p})]\right].\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image38.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image40.png" /></p>
+<blockquote>
+<p>Michigan State University</p>
+</blockquote>
+<blockquote>
+<p>Zhuangdi Zhu and Junyuan Hong and Jiayu Zhou. 2021. Data-Free
+Knowledge Distillation for Heterogeneous Federated Learning. In ICML,
+139:12878-12889.</p>
+</blockquote>
+<h1 id="beyond-sharing-weights-for-deep-domain-adaptation">Beyond
+Sharing Weights for Deep Domain Adaptation</h1>
+<p>2021.10.18</p>
+<h2 id="motivation-3">Motivation</h2>
+<p>Learn features that are invariant to the domain shift.</p>
+<h2 id="method-3">Method</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image41.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image43.png" /></p>
+<h2 id="discussion">Discussion</h2>
+<p>It therefore seems reasonable that thehigher layers of the network,
+which encode higher-level in-formation, should be domain-specific.</p>
+<blockquote>
+<p>Rozantsev, A., Salzmann, M., &amp; Fua, P. (2019). Beyond Sharing
+Weights for Deep Domain Adaptation. IEEE Transactions on Pattern
+Analysis and Machine Intelligence, 41(4), 801-814.</p>
+</blockquote>
+<h1
+id="feddg-federated-domain-generalization-on-medical-image-segmentation-via-episodic-learning-in-continuous-frequency-space">FedDG:
+Federated Domain Generalization on Medical Image Segmentation via
+Episodic Learning in Continuous Frequency Space</h1>
+<p>2021.10.25</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image50.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image48.png" /></p>
+<h2 id="method-4">Method</h2>
 <ul>
-<li>大模型影响序列呈现高度抽象关联</li>
-<li>通过模仿训练数据进行的角色扮演假说</li>
-<li>没有发现支持复杂规划的证据，但无法完全排除这一可能性。</li>
-</ul>
-<h2 id="thinking">Thinking</h2>
+<li>Data
 <ul>
-<li>为啥更大的模型就能学到抽象语义</li>
-<li>逆转诅咒从安全角度，是不是可以作为攻击的一种手段</li>
-<li>模仿，模型要多少条数据才能学会</li>
-</ul>
-<h2 id="method-detail">Method (Detail)</h2>
-<p>每条训练数据对于模型预测的影响分数</p>
+<li>Step 1：对每个客户端的数据都做如右处理</li>
+<li>Step
+2：客户端k的图像X，与傅里叶变换后的数据A（联合任意两个客户端的A）
 <ul>
-<li>换个角度，如果没有这条训练数据，模型预测会发生什么变化？</li>
-<li>所以一种简单直接的思路是，可以把这条数据从训练集中去掉，重新训练一遍模型。:(</li>
-</ul>
-<h3 id="background">Background</h3>
+<li>对X进行傅里叶变换</li>
+<li>把X的振幅分量中的低频分量换为A的低频分量， low_freq_mutate</li>
+<li>最后结合A，做逆傅里叶变换，得到新的图像X’</li>
+</ul></li>
+</ul></li>
+<li>Train
 <ul>
-<li>Step 1: 经验风险最小化</li>
+<li>Loss1:
+根据原始图像X做训练。先利用这里的梯度计算梯度下降后的模型参数F’</li>
+<li>Loss2: 根据F’对X’的输出计算</li>
+<li>Loss3: F与F’ 分别提取mask的边界与背景
+(ndimage.binary_erosion)做对比学习NTXentLoss，loss3*0.1</li>
+</ul></li>
 </ul>
-<p><span class="math display">\[\theta ^ { \star } = \underset { \theta
-\in R ^ { D } } { \operatorname { arg } \operatorname { min } } J (
-\theta , D ) = \underset { \theta \in R ^ { D } } { \operatorname { arg
-} \operatorname { min } } \frac { 1 } { N } \sum _ { i = 1 } ^ { N } L (
-z _ { i } , \theta )\]</span></p>
-<ul>
-<li><p>Step 2: 增加样本 <span class="math inline">\(z_m\)</span>
-后对模型参数的影响， <span class="math inline">\(\epsilon\)</span>
-表示这个样本在训练时的权重</p></li>
-<li><p>Step 3:
-影响函数的<strong>定义</strong>（使用隐函数定理计算）</p></li>
-</ul>
-<p><span class="math display">\[I _ { \theta ^ { \star } } ( z _ { m } )
-= \frac { d \theta ^ { \star } } { d \epsilon } | _ { \epsilon = 0 } = -
-H ^ { - 1 } \nabla _ { \theta } L ( z _ { m } , \theta ^ { \star }
-)\]</span></p>
-<ul>
-<li>Step 4: <strong>H</strong>essian 矩阵→二阶导数</li>
-</ul>
-<p><span class="math display">\[H = \nabla _ { \theta } ^ { 2 } J (
-\theta ^ { \star } , D )\]</span></p>
-<ul>
-<li>Step 5: 所以，样本 z 对模型参数的变化表示为。这个 <span
-class="math inline">\(\epsilon\)</span></li>
-</ul>
-<p><span class="math display">\[\theta ^ { \star } ( \epsilon ) - \theta
-^ { \star } \approx I _ { \theta ^ { \star } } ( z _ { m } ) \epsilon =
-- H ^ { - 1 } \nabla _ { \theta } L ( z _ { m } , \theta ^ { \star } )
-\epsilon\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image51.png" /></p>
+<p>解决从多个分布域中学习的问题-&gt;数据异质问题</p>
+<p>一般的深度学习中，数据是共享的，所以可以直接进行跨域学习。然而在联邦学习中，数据并不支持直接共享，所以从数据分布角度入手。从原始图像中抽取中分布(风格)信息，并把这个信息共享出来。这里假设通过这个分布信息无法还原真实数据。How？</p>
+<p>通过傅里叶变换，把数据转换为：振幅谱+相位谱。下图的蓝色背景部分，通过公式进行转换后，再把各个客户端的信息数据进行汇总学习。</p>
+<p>由于训练背景是医学图像处理，涉及到语义分割，所以需要使用常规的分割损失函数。再考虑图形的边界与背景，对应右边的绿色部分，采用InfoNCE损失函数
+对边界与背景做提取处理，计算InfoNCE损失。hm，hp表示同一类的正面特征，表示边界与背景。</p>
+<p>所以整个训练过程就是利用傅里叶变换提取分布信息，结合一般的分割与InfoNCE损失进行训练。主要的创新点在傅里叶变换提取了可共享的信息。下一次再进行具体的方法介绍。</p>
+<blockquote>
+<p>CUHK, 香港中文大学</p>
+</blockquote>
+<blockquote>
+<p>Liu Q, Chen C, Qin J, et al. Feddg: Federated domain generalization
+on medical image segmentation via episodic learning in continuous
+frequency space. Proceedings of the IEEE/CVF Conference on Computer
+Vision and Pattern Recognition. 2021: 1013-1023.</p>
+</blockquote>
+<h1
+id="refine-myself-by-teaching-myself-feature-refinement-via-self-knowledge-distillation">Refine
+Myself by Teaching Myself : Feature Refinement via Self-Knowledge
+Distillation</h1>
+<p>2021.11.11</p>
+<h2 id="method-5">Method</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image53.png" /></p>
+<p><span class="math display">\[
+\begin{array}{l}
+\mathcal{L}_{FRSKD}(\bm{x},y;\theta_{c},\theta_{t},K) =
+\mathcal{L}_{CE}(\bm{x},y;\theta_{c})+\mathcal{L}_{C
+E}(\bm{x},y;\theta_{t}) +\alpha\cdot\mathcal{L}_{K
+D}(\bm{x};\theta_{c},\theta_{t},K)+\beta\cdot\mathcal{L}_{F}(T,F;\theta_{c},\theta_{t})
+\end{array}
+\]</span></p>
+<p><span
+class="math display">\[\mathcal{L}_{F}(T,F;\theta_{c},\theta_{t})=\Sigma_{i=1}^{n}\vert\vert\phi(T_{i})-\phi(F_{i})\vert\vert_2\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image56.png" /></p>
+<h2 id="result-1">Result</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image57.png" /></p>
+<p><a
+href="https://github.com/MingiJi/FRSKD">https://github.com/MingiJi/FRSKD</a></p>
+<blockquote>
+<p>Ji M, Shin S, Hwang S, et al. Refine Myself by Teaching Myself:
+Feature Refinement via Self-Knowledge Distillation. Proceedings of the
+IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2021:
+10664-10673.</p>
+</blockquote>
+<h1
+id="no-fear-of-heterogeneity-classifier-calibration-for-federated-learning-with-non-iid-data">No
+Fear of Heterogeneity: Classifier Calibration for Federated Learning
+with Non-IID Data</h1>
+<p>2021.11.25</p>
+<h2 id="motivation-4">Motivation</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image58.png" /></p>
+<h2 id="method-6">Method</h2>
+<p><span
+class="math display">\[\mu_{c,k}=\frac{1}{N_{c,k}}\sum_{j=1}^{N_{c,k}}z_{c,k,j},\quad\Sigma_{c,k}=\frac{1}{N_{c,k}-1}\sum_{j=1}^{N_{c,k}}\left(z_{c,k,j}-\mu_{c,k}\right)\left(z_{c,k,j}-\mu_{c,k}\right)^{T}\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image61.png" /></p>
+<h2 id="result-2">Result</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image62.png" /></p>
+<blockquote>
+<p>Luo M, Chen F, Hu D, et al. No Fear of Heterogeneity: Classifier
+Calibration for Federated Learning with Non-IID Data[J]. arXiv preprint
+arXiv:2106.05001, 2021.</p>
+</blockquote>
+<h1 id="federated-learning-with-personalization-layers">Federated
+Learning with Personalization Layers</h1>
+<p>2021.12.02</p>
+<h2 id="method-7">Method</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image64.png" /></p>
+<p>Personalized population risk</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image65.png" /></p>
+<p>Empirical risk</p>
+<p><span class="math display">\[L_{j}^{E
+R}\big(\displaystyle\mathrm{W}_B,\displaystyle\mathrm{W}_{P}\big)\triangleq\frac{1}{n_{j}}\sum_{i=1}^{n_{j}}l(y_{j,i},f\big(x_{j,i};\displaystyle\mathrm{W}_B,\displaystyle\mathrm{W}_{P}\big)\big)\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image67.png" /></p>
+<blockquote>
+<p>Manoj Ghuhan Arivazhagan, Vinay Aggarwal, Aaditya Kumar Singh, and
+Sunav Choudhary. 2019. Federated learning with personalization layers.
+arXiv preprint arXiv:1912.00818 (2019).</p>
+</blockquote>
+<h1
+id="federated-split-vision-transformer-for-covid-19-cxr-diagnosis-using-task-agnostic-training">Federated
+Split Vision Transformer for COVID-19 CXR Diagnosis using Task-Agnostic
+Training</h1>
+<p>2021.12.09</p>
+<h2 id="motivation-5">Motivation</h2>
+<p>融合联邦学习与分割学习尽可能发挥他们的独特优势</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image69.png" /></p>
+<h2 id="method-8">Method</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image70.png" /></p>
+<blockquote>
+<p>Park S, Kim G, Kim J, et al. Federated Split Vision Transformer for
+COVID-19 CXR Diagnosis using Task-Agnostic Training[J]. arXiv preprint
+arXiv:2111.01338, 2021.</p>
+</blockquote>
+<h1
+id="local-learning-matters-rethinking-data-heterogeneity-in-federated-learning">Local
+Learning Matters: Rethinking Data Heterogeneity in Federated
+Learning</h1>
+<p>2022.03.18</p>
+<h2 id="contribution">Contribution</h2>
+<p>Focus on local learning generality rather than proximal
+restriction</p>
+<h2 id="method-9">Method</h2>
+<h3 id="stochastic-depth-2016">Stochastic Depth (2016)</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image71.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> <span class="keyword">not</span> <span class="variable language_">self</span>.training <span class="keyword">or</span> torch.rand(<span class="number">1</span>)[<span class="number">0</span>] &gt;= <span class="variable language_">self</span>.death_rate:</span><br><span class="line">    ……</span><br><span class="line">     <span class="keyword">if</span> <span class="variable language_">self</span>.training:</span><br><span class="line">         residual /= (<span class="number">1.</span> - <span class="variable language_">self</span>.death_rate)</span><br><span class="line">    ……</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h3 id="mixup-2018">Mixup (2018)</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image72.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">index = torch.randperm(batch_size)</span><br><span class="line">mixed_x = lam * x + (<span class="number">1</span> - lam) * x[index, :]</span><br><span class="line">y_a, y_b = y, y[index]</span><br><span class="line">……</span><br><span class="line">loss = lam * f(pred, y_a) + (<span class="number">1</span> - lam) * f(pred, y_b)</span><br></pre></td></tr></table></figure>
+<h3 id="gradaug-2020">GradAug (2020)</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image73.png" /></p>
+<p>Random number of channels in each layer</p>
+<blockquote>
+<p>University of Central Florida</p>
+</blockquote>
+<blockquote>
+<p>Mendieta M, Yang T, Wang P, et al. Local Learning Matters: Rethinking
+Data Heterogeneity in Federated Learning[J]. arXiv preprint
+arXiv:2111.14213, 2021.</p>
+</blockquote>
+<h1
+id="local-learning-matters-rethinking-data-heterogeneity-in-federated-learning-1">Local
+Learning Matters: Rethinking Data Heterogeneity in Federated
+Learning</h1>
+<p>2022.03.25</p>
+<h2 id="motivation-6">Motivation</h2>
 <ul>
-<li>Step 6:
-由于很难解释整个参数变化的影响，通常会固定对某个输入的影响，即输出的
-logits</li>
+<li><p>Promote smooth optimization and consistency within the
+model</p></li>
+<li><p>Reduce computation in a purposeful manner</p></li>
 </ul>
-<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx - \nabla
-_ { \theta } f ( \theta ^ { s } ) ^ { T } ( G + \lambda I ) ^ { - 1 }
-\nabla _ { \theta } L ( z _ { m } , \theta ^ { s } )
-I _ { f } ( z _ { m } ) = \nabla _ { \theta } f ( \theta ^ { \star } ) ^
-{ T } I _ { \theta ^ { \star } } ( z _ { m } ) = - \nabla _ { \theta } f
-( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla _ { \theta } L ( z _ {
-m } , \theta ^ { \star } )\]</span></p>
+<h2 id="method-10">Method</h2>
 <ul>
-<li>Step 7: 最后，衡量样本 z 的效果 → 对于某个输出，样本 z
-的影响分数</li>
+<li><p>Untrained network in place of the traditional logit-based
+loss</p></li>
+<li><p>Reuse the intermediate features of the full network</p></li>
 </ul>
-<p><span class="math display">\[f ( \theta ^ { \star } ( \epsilon ) ) -
-f ( \theta ^ { \star } ) \approx I _ { f } ( z _ { m } ) \epsilon = -
-\nabla _ { \theta } f ( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla
-_ { \theta } L ( z _ { m } , \theta ^ { \star } ) \epsilon\]</span></p>
-<p><a
-href="https://arxiv.org/abs/2308.03296">https://arxiv.org/abs/2308.03296</a></p>
-<h3 id="previou-work">Previou Work</h3>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image75.png" /></p>
+<h2 id="result-3">Result</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image74.png" /></p>
+<blockquote>
+<p>University of Central Florida</p>
+</blockquote>
+<blockquote>
+<p>Mendieta M, Yang T, Wang P, et al. Local Learning Matters: Rethinking
+Data Heterogeneity in Federated Learning[J]. arXiv preprint
+arXiv:2111.14213, 2021.</p>
+</blockquote>
+<h1 id="motion-representations-for-articulated-animation">Motion
+Representations for Articulated Animation</h1>
+<p>2022.04.15</p>
+<h2 id="问题">问题</h2>
 <ul>
-<li><p><a href="https://arxiv.org/abs/1703.04730">Influence
-functions</a> are a classic technique from statistics for determining
-which training examples contribute significantly to a model’s
-outputs.</p>
+<li>输入：图片与视频</li>
+<li>目标：图片中的目标“动”起来</li>
+<li>挑战：目标与背景提取；视频帧与帧之间的关联 ## 建模：</li>
+<li>对图片与每帧视频进行粗估计，提取驱动的目标
 <ul>
-<li>how would that change the trained parameters (and, by extension, the
-model’s outputs)?</li>
-<li>The “influence” of a training example is an approximation to how it
-affects the final parameters. Most often, we start with some measure of
-interest (such as the probability the model assigns to a given response)
-and attempt to identify the training examples that are most
-influential.</li>
+<li>AutoEncoder（AE）：提取图片的feature map（FM）</li>
+<li>单层卷积：根据FM提取驱动的目标区域（限定好区域大小），R_p, V_p</li>
+<li>计算仿射参数（PCA 或 Jacobian）</li>
+<li>若干层下采样+单层全连接（特征点数）（拼接后输入）</li>
 </ul></li>
-<li><p>Except</p>
+<li>图片生成（细粒度）
 <ul>
-<li>if the models responded to user prompts by splicing together
-sequences from the training set, then we’d expect the influential
-sequences for a given model response to include expressions of
-near-identical thoughts.</li>
-<li>influential sequences related at a more abstract thematic level
-would be a sign that the model has acquired higher-level concepts or
-representations</li>
+<li>原图：Block(卷积+下采样)+ResBlock</li>
+<li>AE+卷积（区域数）</li>
+<li>上采样+卷积</li>
 </ul></li>
 </ul>
-<p><a
-href="https://arxiv.org/abs/1703.04730">https://arxiv.org/abs/1703.04730</a></p>
-<h3 id="optimize">Optimize</h3>
-<p>公式</p>
-<p><span class="math display">\[f ( \theta ^ { \star } ( \epsilon ) ) -
-f ( \theta ^ { \star } ) \approx I _ { f } ( z _ { m } ) \epsilon = -
-\nabla _ { \theta } f ( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla
-_ { \theta } L ( z _ { m } , \theta ^ { \star } ) \epsilon\]</span></p>
-<p><span class="math display">\[H = \nabla _ { \theta } ^ { 2 } J (
-\theta ^ { \star } , D )\]</span></p>
-<p>有两个影响计算耗时的地方，文章对这两个地方进行了优化</p>
-<ul>
-<li>海森矩阵，二阶导数计算（模型相关）→ 用现成的优化算法</li>
-</ul>
-<ol type="1">
-<li>迭代法</li>
-<li>Kronecker-Factored Approximate Curvature
-<strong>(K-FAC)</strong></li>
-</ol>
-<ul>
-<li>训练数据集特别大（数据相关）→ 过滤数据</li>
-</ul>
-<ol type="1">
-<li>Step 1：TF-IDF
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image77.png" /></p>
+<h2 id="结果">结果</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image76.gif" /></p>
+<blockquote>
+<p>University of Trento, Italy</p>
+</blockquote>
+<blockquote>
+<p>Siarohin, Aliaksandr, Oliver, Woodford, Jian, Ren, Menglei, Chai, and
+Sergey, Tulyakov. "Motion Representations for Articulated Animation." .
+In CVPR.2021.</p>
+</blockquote>
+<h1
+id="communication-efficient-federated-learning-via-knowledge-distillation">Communication-efficient
+federated learning via knowledge distillation</h1>
+<p>2022.05.06</p>
+<h2 id="method-11">Method</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image78.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image79.png" /></p>
+<p>SVD energy 随着轮数增加，阈值需要增大，即重要的知识越来越多</p>
+<p><span class="math display">\[{\mathcal
+L}_{t,i}^{d}=\frac{\mathrm{KL}({\bf y}_{i}^{s}{\bf
+y}_{i}^{t})}{\mathcal{L}_{t,i}^{t}+\mathcal{L}_{s,i}^{t}}\]</span></p>
+<p><span
+class="math display">\[\mathcal{L}_{t,i}^{t}=\mathrm{CE}(\mathrm{y}_{i},\mathrm{y}_{i}^{t})\]</span></p>
+<p><span
+class="math display">\[\mathcal{L}_{t,i}^{h}=\mathcal{L}_{s,i}^{h}=\frac{\mathrm{MSE}({\mathbf
+H}_i^t,{\mathbf W}_{i}^{h}H^{s})+\mathrm{MSE}({\bf A}_{i}^{t},{\bf
+A}^{s})}{\mathcal{L}_{t,i}^{t}+\mathcal{L}_{s,i}^{t}}\]</span></p>
+<p><span
+class="math display">\[T(t)=T_{\mathrm{start}}+(T_{\mathrm{end}}-T_{\mathrm{start}})t,t\in[0,1]\]</span></p>
 <ol type="1">
-<li>计算 query 中每个 token 的重要性分数</li>
-<li>doc 的 TF-IDF 分数，只将所有 token 的重要性分数相加</li>
-<li>Okapi BM25 <span class="math display">\[\operatorname { score } ( Q
-, D ) = \sum _ { t = 1 } ^ { T } \frac { ( k _ { 1 } + 1 ) \times  {\rm
-exists\_in\_doc } ( t _ { t } , D ) } { k _ { 1 } +
-{\rm  exists\_in\_doc } ( t _ { t } , D ) } \operatorname { IDF } ( t _
-{ t } )\]</span> <span class="math display">\[\operatorname { IDF } ( t
-) = \operatorname { log } ( \frac { C - \operatorname { count } ( t ) +
-0.5 } { \operatorname { count } ( t ) + 0.5 } + 1 )\]</span></li>
-<li>选前 1w 个序列作为候选集（相当于把这些作为训练集 <em>D</em>）</li>
-</ol></li>
-<li>Step 2：Query Batching</li>
+<li><p>Client: Layer SVD-&gt;u*sigma*v</p></li>
+<li><p>Server: mean(u*sigma*v)-&gt; u*sigma*v</p></li>
+<li><p>Client Revice: replace u*sigma*v</p></li>
 </ol>
-<p>注：</p>
-<ul>
-<li>重要性得分：随着 token 在 query
-中出现的次数增加而增加，随着它在整个语料库中出现的 doc
-数量减少而减少</li>
-</ul>
-<h3 id="equation-to-code">Equation to Code</h3>
-<ul>
-<li>影响力计算公式</li>
-</ul>
-<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx - \nabla
-_ { \theta } f ( \theta ^ { s } ) ^ { T } ( G + \lambda I ) ^ { - 1 }
-\nabla _ { \theta } L ( z _ { m } , \theta ^ { s } )\]</span></p>
+<blockquote>
+<p>Tsinghua University</p>
+</blockquote>
+<blockquote>
+<p>Wu, C., Wu, F., Lyu, L. et al. Communication-efficient federated
+learning via knowledge distillation. Nat Commun 13, 2032 (2022)</p>
+</blockquote>
+<h1
+id="personalized-federated-learning-using-hypernetworks">Personalized
+Federated Learning using Hypernetworks</h1>
+<p>2022.06.24</p>
+<h2 id="method-12">Method</h2>
+<p>Using a single joint hypernetwork to generate all separate models
+allows us to perform smart parameter sharing.</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image84.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image85.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image87.png" /></p>
+<h2 id="results">Results</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image86.png" /></p>
+<blockquote>
+<p>Shamsian A, Navon A, Fetaya E, et al. Personalized federated learning
+using hypernetworks[C]//International Conference on Machine Learning.
+PMLR, 2021: 9489-9502.</p>
+</blockquote>
+<h1 id="the-power-of-scale-for-parameter-efficient-prompt-tuning">The
+Power of Scale for Parameter-Efficient Prompt Tuning</h1>
+<p>2023.04.04</p>
+<h2 id="motivation-7">Motivation</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image88.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image91.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image90.png" /></p>
+<h2 id="method-13">Method</h2>
 <ul>
-<li><span class="math inline">\(z_m\)</span> 表示一条训练数据</li>
-<li><span class="math inline">\(\theta^s\)</span> 表示训练后的模型（在
-<span class="math inline">\(z_m\)</span> 组成的训练集上）</li>
-<li><span class="math inline">\(\mathbf{G}+\lambda \mathbf{I}\)</span>
-可以简单理解为优化计算复杂度后的 Hessian 矩阵（二阶导数）</li>
-<li>Claude 帮忙写的伪代码</li>
+<li><p>模型输入层额外加入一层Embedding，其他部分冻结</p></li>
+<li><p>额外的数据输入处理</p></li>
 </ul>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">model = LLM()  <span class="comment"># 经过训练的模型</span></span><br><span class="line">criterion = nn.CrossEntropyLoss()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 需要查询的输入和输出</span></span><br><span class="line">input_ = <span class="string">&quot;感冒了该怎么办&quot;</span></span><br><span class="line">target = <span class="string">&quot;多喝水,好好休息&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 模型输出</span></span><br><span class="line">output = model(input_)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 海森矩阵</span></span><br><span class="line">hessian = torch.autograd.functional.hessian(output, model.parameters())</span><br><span class="line"></span><br><span class="line"><span class="comment"># 对loss求导得到模型参数梯度</span></span><br><span class="line">loss = criterion(output, target)</span><br><span class="line">loss.backward()</span><br><span class="line">gradient = [p.grad <span class="keyword">for</span> p <span class="keyword">in</span> model.parameters()]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 遍历训练数据,计算每个数据对loss的影响</span></span><br><span class="line">max_influences = []</span><br><span class="line"><span class="keyword">for</span> x, y <span class="keyword">in</span> dataloader:</span><br><span class="line">    <span class="comment"># 前向传播</span></span><br><span class="line">    output = model(x)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 对样本x,y求导</span></span><br><span class="line">    influence_loss = criterion(output, y)</span><br><span class="line">    influence_loss.backward(retain_graph=<span class="literal">True</span>)</span><br><span class="line">    influence_grad = [p.grad <span class="keyword">for</span> p <span class="keyword">in</span> model.parameters()]</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 利用海森矩阵和梯度计算influence</span></span><br><span class="line">    <span class="comment"># -sample_grad * H * zm_grad</span></span><br><span class="line">    influence = -torch.dot(hessian, influence_grad) * gradient</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 记录影响最大的数据</span></span><br><span class="line">    max_influences.append((x, y, influence))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 返回影响最大的几个训练数据</span></span><br><span class="line">max_influences = <span class="built_in">sorted</span>(max_influences, key=<span class="keyword">lambda</span> x: x[<span class="number">2</span>], reverse=<span class="literal">True</span>)[:<span class="number">5</span>]</span><br></pre></td></tr></table></figure>
-<h3 id="attribution-to-layers-and-tokens">Attribution to Layers and
-Tokens</h3>
-<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx - \nabla
-_ { \theta } f ( \theta ^ { s } ) ^ { T } ( G + \lambda I ) ^ { - 1 }
-\nabla _ { \theta } L ( z _ { m } , \theta ^ { s } )\]</span></p>
-<p>拆分 <span class="math inline">\(\theta^s\)</span> 为每一层</p>
-<p><span class="math display">\[q = - \nabla _ { \theta } f ( \theta ^ {
-s } )\]</span></p>
-<p><span class="math display">\[r = \nabla _ { \theta } L ( z _ { m } ,
-\theta ^ { s } )\]</span></p>
-<p>拆分到层</p>
-<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx q ^ { T
-} ( \hat { G } + \lambda I ) ^ { - 1 } r = \sum _ { \ell = 1 } ^ { L } q
-_ { \ell } ^ { T } ( \hat { G } _ { \ell } + \lambda I ) ^ { - 1 } r _ {
-\ell }\]</span></p>
-<p>而对于每条训练数据 <span class="math inline">\(z_m\)</span> ，<span
-class="math inline">\(r\)</span> 可以更进一步的拆分为每个 token ( <span
-class="math inline">\(r =\sum_t r_t\)</span> )，所以有</p>
-<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx \sum _ {
-\ell = 1 } ^ { L } \sum _ { t = 1 } ^ { T } q _ { \ell } ^ { T } ( \hat
-{ G } _ { \ell } + \lambda I ) ^ { - 1 } r _ { \ell , t }\]</span></p>
-<p>但是对于 token 层级来说，每个 token
-都是包含整条数据的信息（之前所有输入的 token），所以并不能独立观察。</p>
-<p>额外说明：如果 Predident George Washington 具有影响力，因为预测了
-George，则 President （前一个 token）将会高亮显示。</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/0b2bafe670a7427a90200dd60de5aa83.webp" /></p>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image93.png" /></p>
+<h2 id="result-4">Result</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/FL_papers/image92.png" /></p>
+<blockquote>
+<p>Google</p>
+</blockquote>
+<blockquote>
+<p>Lester B , et al. The Power of Scale for Parameter-Efficient Prompt
+Tuning. In EMNLP. 2021:3045–3059</p>
+</blockquote>
 ]]></content>
       <categories>
         <category>PaperReading</category>
       </categories>
       <tags>
-        <tag>NLP</tag>
-        <tag>Interpretability</tag>
+        <tag>Federated Learning</tag>
       </tags>
   </entry>
   <entry>
-    <title>vLLM  的 PageAttention</title>
-    <url>/hexo_blog/2023/11/24/vLLM%E7%9A%84PageAttention/</url>
-    <content><![CDATA[<p>介绍 vLLM 的 PageAttention 加速</p>
-<p><a href="https://blog.vllm.ai/2023/06/20/vllm.html">博客</a></p>
-<p><a href="https://arxiv.org/pdf/2309.06180.pdf">论文</a></p>
+    <title>Hello World</title>
+    <url>/hexo_blog/1970/01/01/hello-world/</url>
+    <content><![CDATA[<p>Welcome to <a href="https://hexo.io/">Hexo</a>! This is your very
+first post. Check <a href="https://hexo.io/docs/">documentation</a> for
+more info. If you get any problems when using Hexo, you can find the
+answer in <a
+href="https://hexo.io/docs/troubleshooting.html">troubleshooting</a> or
+you can ask me on <a
+href="https://github.com/hexojs/hexo/issues">GitHub</a>.</p>
+<span id="more"></span>
+<h2 id="quick-start">Quick Start</h2>
+<h3 id="create-a-new-post">Create a new post</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">$ hexo new <span class="string">&quot;My New Post&quot;</span></span><br></pre></td></tr></table></figure>
+<p>More info: <a
+href="https://hexo.io/docs/writing.html">Writing</a></p>
+<h3 id="run-server">Run server</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">$ hexo server</span><br></pre></td></tr></table></figure>
+<p>More info: <a href="https://hexo.io/docs/server.html">Server</a></p>
+<h3 id="generate-static-files">Generate static files</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">$ hexo generate</span><br></pre></td></tr></table></figure>
+<p>More info: <a
+href="https://hexo.io/docs/generating.html">Generating</a></p>
+<h3 id="deploy-to-remote-sites">Deploy to remote sites</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">$ hexo deploy</span><br></pre></td></tr></table></figure>
+<p>More info: <a
+href="https://hexo.io/docs/deployment.html">Deployment</a></p>
+]]></content>
+      <categories>
+        <category>test</category>
+      </categories>
+      <tags>
+        <tag>test</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Tracing Model Outputs to the Training Data</title>
+    <url>/hexo_blog/2023/11/17/Tracing-Model-Outputs-to-the-Training-Data/</url>
+    <content><![CDATA[<p><a href="https://arxiv.org/abs/2308.03296">arxiv</a></p>
 <p><a
-href="https://github.com/vllm-project/vllm/blob/7c600440f7560348e571f021f2b2d1469de5264d/csrc/attention/attention_kernels.cu#L71-L383">代码</a></p>
+href="https://www.anthropic.com/index/influence-functions">blog</a></p>
+<p><a
+href="https://gamma.app/docs/Tracing-Model-Outputs-to-the-Training-Data-xs3bsz5n91p95ms?mode=doc">pptx</a></p>
 <span id="more"></span>
-<p>文中 GIF 图来源于 vLLM 博客</p>
-<h2 id="动机">动机</h2>
-<h3 id="定义">定义</h3>
-<p>当前模型生成的机理是，给定 Prompt，输出
-Completion。模型在生成时有一个参数
-MAX_SEQ_LEN，以控制模型最大能处理的长度（Prompt + Completion）。</p>
-<h3 id="问题">问题</h3>
-<p>经典的框架在生成时，会提前申请 MAX_SEQ_LEN 的长度。然而，最后生成的
-Prompt + Completion 的长度往往可能是小于 MAX_SEQ_LEN
-的。从而导致有很大显存的浪费，且生成之后，该部分显存又将丢弃，从而导致操作的浪费。</p>
-<h3 id="小结">小结</h3>
-<p>传统方法在生成时，会有申请/丢弃显存的 <strong>IO
-浪费</strong>，模型在处理时会有冗余显存始终未被利用到，导致模型能处理的
-Prompt 低于理想情况，<strong>计算浪费</strong>。</p>
-<h2 id="怎么做">怎么做</h2>
-<p>冗余的部分来自于，attention
-计算中的空间。因此，借鉴操作系统中虚拟内存和分页的经典思想，将空间打碎，从而实现更高效的空间利用。</p>
-<p>翻译后的原句：PagedAttention
-允许在非连续的内存空间中存储连续的键和值。具体来说，PagedAttention
-将每个序列的 KV
-缓存划分为块，每个块包含固定数量的标记的键和值。在注意力计算过程中，PagedAttention
-内核有效地识别并获取这些块。</p>
-<p>kv cache 空间计算：</p>
-<p><span class="math inline">\(2 \times 5120 \times 40 \times 2 \times
-2048 / (1024 ** 3)\)</span> GB = 1.5625 GB</p>
-<p>(k+v) <span class="math inline">\(\times\)</span> hidden_size <span
-class="math inline">\(\times\)</span> num_layers <span
-class="math inline">\(\times\)</span> FP16 bytes <span
-class="math inline">\(\times\)</span> tokens_num</p>
-<p>对于 13B 模型 2048 的 seq_len 而言，需要申请 1.56 GB 的显存。而如果
-Prompt + Completion 的长度只有 1024，那么将浪费 0.78 G
-的显存（在没有其他 kv 优化的情况下）</p>
-<p>如下图所示，block 存储了所有可能的 token。在生成时，模型从 block
-中进行计算。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/vllm_pageattention/annimation0.gif" /></p>
-<p>Attention 计算公式</p>
-<p><span class="math display">\[a _ { i j } = \frac { \operatorname {
-exp } ( q _ { i } ^ { T } k _ { j } / \sqrt { d } ) } { \sum _ { t = 1 }
-^ { i } \operatorname { exp } ( q _ { i } ^ { T } k _ { t } / \sqrt { d
-} ) } , o _ { i } = \sum _ { j = 1 } ^ { i } a _ { i j } v _ { j
-}\]</span></p>
-<p>PageAttention 计算公式</p>
-<p><span class="math display">\[A _ { i j } = \frac { \operatorname {
-exp } ( q _ { i } ^ { T } K _ { j } / \sqrt { d } ) } { \sum _ { t = 1 }
-^ { \lceil i / B \rceil } \operatorname { exp } ( q _ { i } ^ { T } K _
-{ t } 1 / \sqrt { d } ) } , o _ { i } = \sum _ { j = 1 } ^ { \lceil i /
-B \rceil } V _ { j } A _ { i j } ^ { T }\]</span></p>
-<p>对于 query (q)，不需要改变，而 k 和 v 转变为 block
-的形式。计算步骤如下</p>
-<ol type="1">
-<li>提取对应的 block</li>
-<li>计算 q 和 block 中每个元素的结果，汇总会注意力分数 A</li>
-<li>注意力分数 A 与 block 中 V 相乘，得到输出</li>
-</ol>
-<h2 id="细节实现">细节实现</h2>
-<p>如下图所示，kv cache 分了 logical block 和 physical
-block，前者和后者内容是一样的，通过 Block table
-找到在显存中的实际位置。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/vllm_pageattention/annimation1.gif" /></p>
-<p>而如果是多个句子同时请求，那么在显存足够的情况下，多个句子会同时处理。如下所示，每个句子有自己的
-logical block，共同映射到 physical block 中。（多个句子会一起计算）</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/vllm_pageattention/annimation3.gif" /></p>
-<h2 id="代码实现">代码实现</h2>
-<p>调度是通过 Python
-实现的，这一部分逻辑有些杂乱，之后再看。这里主要关注 PageAttention
-的计算，核心的代码如下</p>
-<h3 id="背景知识">背景知识</h3>
-<p><code>#pragma unroll</code>
-控制了其紧跟着的下一个循环的展开（为了加速使用，下面将删除对应行）</p>
-<p>__syncthreads();
-是CUDA并行编程中的一个同步指令，用于确保在同一个线程块内的所有线程都到达此指令位置后才能继续执行后面的代码，意味着在这个函数之前的所有操作都已经完成，buffer或者内存的数据已经得到同步更新。（下面将删除对应行）</p>
-<p>__shfl_xor_sync：进行数据交换和归约操作</p>
-<p>__shfl_sync：广播操作</p>
-<p><code>WARP_SIZE：</code>在CUDA编程中，<code>WARP_SIZE</code>通常指的是一个warp中的线程数。warp是CUDA硬件的一个基本执行单元，每个warp包含了一定数量的线程，这些线程会同时执行相同的指令。<code>WARP_SIZE 通常定义为</code>32，这意味着每个warp包含32个线程。</p>
-<p>逐步拆建，首先，确定这段函数在干啥。它代替了 Python
-的以下计算，简单来说有两个矩阵乘法，以及一个 softmax
-操作。涉及到的变量有 q、k、v。（忽略 mask）</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">score = torch.matmul(q, k.transpose(<span class="number">2</span>, <span class="number">3</span>)) / math.sqrt(<span class="variable language_">self</span>.head_dim)</span><br><span class="line">score = nn.functional.softmax(score, dim=-<span class="number">1</span>, dtype=torch.float32).to(q.dtype)</span><br><span class="line">out = torch.matmul(score, v)</span><br></pre></td></tr></table></figure>
-<p>之后，再捋一捋变量的数据结构，从下面的代码开始，假设 q、k、v 的
-num_head 和 head_size 是相同的</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Copy from https://www.dipkumar.dev/becoming-the-unbeatable/posts/gpt-kvcache/</span></span><br><span class="line"> </span><br><span class="line"><span class="comment"># 一行 Attention 的版本</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">attention</span>(<span class="params">q, k, v, mask</span>):</span><br><span class="line">    <span class="keyword">return</span> softmax(q @ k.T / np.sqrt(q.shape[-<span class="number">1</span>]) + mask) @ v</span><br><span class="line">   </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func</span>(<span class="params">x, kvcache</span>):</span><br><span class="line">    <span class="comment"># n 条句子，每个句子有 n_embd 的向量，分别经过 qkv 的 mlp，可以得到每个句子的 q、k、v</span></span><br><span class="line">    <span class="comment"># 其维度分别为 [n_seq, n_embd]</span></span><br><span class="line">    x = mlp(x, **c_attn)  <span class="comment"># [n_seq, n_embd] -&gt; [n_seq, 3*n_embd]</span></span><br><span class="line">    qkv = np.split(x, <span class="number">3</span>, axis=-<span class="number">1</span>)  <span class="comment"># [n_seq, 3*n_embd] -&gt; [3, n_seq, n_embd]</span></span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 在第 i-1 个 token 的生成过程中，为了加速会利用之前生成 token 的 kv_cache。为什么？因为要在之前的 token 中查询</span></span><br><span class="line">    <span class="comment"># 所以，实际上 k 和 v 的结构为 [prev_n_seq+n_seq, n_embd]</span></span><br><span class="line">    <span class="keyword">if</span> kvcache:</span><br><span class="line">        new_q, new_k, new_v = qkv</span><br><span class="line">        old_k, old_v = kvcache</span><br><span class="line">        k = np.vstack([old_k, new_k])</span><br><span class="line">        v = np.vstack([old_v, new_v])</span><br><span class="line">        qkv = [new_q, k, v]</span><br><span class="line"> </span><br><span class="line">    current_cache = [qkv[<span class="number">1</span>], qkv[<span class="number">2</span>]]</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># split into heads</span></span><br><span class="line">    <span class="comment"># 这里又需要根据 head 来切分，所以结构更新为</span></span><br><span class="line">    <span class="comment"># q: [n_head, n_seq,            n_embd/n_head]</span></span><br><span class="line">    <span class="comment"># k: [n_head, prev_n_seq+n_seq, n_embd/n_head]</span></span><br><span class="line">    <span class="comment"># v: [n_head, prev_n_seq+n_seq, n_embd/n_head]</span></span><br><span class="line">    qkv_heads = <span class="built_in">list</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: np.split(x, n_head, axis=-<span class="number">1</span>), qkv))  <span class="comment"># [3, n_seq, n_embd] -&gt; [n_head, 3, n_seq, n_embd/n_head]</span></span><br><span class="line"> </span><br><span class="line">    <span class="comment"># causal mask to hide future inputs from being attended to</span></span><br><span class="line">    <span class="keyword">if</span> kvcache:</span><br><span class="line">        causal_mask = np.zeros((<span class="number">1</span>, k.shape[<span class="number">0</span>]))</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        causal_mask = (<span class="number">1</span> - np.tri(x.shape[<span class="number">0</span>])) * -<span class="number">1e10</span>  <span class="comment"># [n_seq, n_seq]</span></span><br><span class="line"> </span><br><span class="line">    <span class="comment"># perform attention over each head</span></span><br><span class="line">    out_heads = [attention(q, k, v, causal_mask) <span class="keyword">for</span> q, k, v <span class="keyword">in</span> <span class="built_in">zip</span>(*qkv_heads)]  <span class="comment"># [n_head, 3, n_seq, n_embd/n_head] -&gt; [n_head, n_seq, n_embd/n_head]</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<p>而在 PageAttention 中，k 和 v 分别用 block
-再拆了一遍，所以需要多两个维度 num_blocks 以及
-block_size。并且，为了实现连续 Batch，n_seq 将会被拼成一个 seq
-处理。所以最后结构如下：</p>
+<h2 id="motivation"><strong>Motivation</strong></h2>
 <ul>
-<li>q: [n_seq, n_head, head_size]</li>
-<li>k: [num_blocks, n_head, head_size, block_size]</li>
-<li>v: [num_blocks, n_head, head_size, block_size]</li>
+<li>Understanding the inner workings of language models will have
+substantial implications for forecasting AI capabilities as well as for
+approaches to aligning AI systems with human preferences.</li>
+<li>了解模型内部工作机理能够更好地让模型<strong>对齐</strong>人类偏好</li>
 </ul>
-<p>这里的 head_size = n_embd/n_head，且调换了一些维度。</p>
-<p>而对于输出 out: [n_seq, n_head, head_size]</p>
-<p>接下来看关键实现。假设有两个请求，seq1 = [1, 2, 3], seq2 = [1, 3, 4,
-5]。vLLM 会将其拼成一个 seq = [1,2,3,1,3,4,5]</p>
-<p>取 q</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 这里的传参 q // [num_seqs, num_heads, head_size]</span></span><br><span class="line"><span class="comment">// 首先是转成了指针的形式，其次是用了线程，以方便 CUDA 进行加速</span></span><br><span class="line"> </span><br><span class="line"><span class="comment">// seq_idx 表示第 idx 条句子，q_stride 表示这条句子的长度</span></span><br><span class="line"><span class="comment">// head_idx 表示第 idx 个head，HEAD_SIZE 是 head 的大小</span></span><br><span class="line">  <span class="type">const</span> <span class="type">scalar_t</span>* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// THREAD_GROUP_SIZE：线程组的大小；NUM_VECS_PER_THREAD：表示每个线程处理的向量数量（并行）</span></span><br><span class="line">  __shared__ Q_vec q_vecs[THREAD_GROUP_SIZE][NUM_VECS_PER_THREAD];</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = thread_group_idx; i &lt; NUM_VECS_PER_THREAD; i += NUM_THREAD_GROUPS) &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;</span><br><span class="line">    q_vecs[thread_group_offset][i] = *<span class="built_in">reinterpret_cast</span>&lt;<span class="type">const</span> Q_vec*&gt;(q_ptr + vec_idx * VEC_SIZE);</span><br><span class="line">  &#125;</span><br><span class="line"><span class="comment">// 使用 q_vecs 来存储</span></span><br></pre></td></tr></table></figure>
-<p>取 k 和 第一个矩阵乘法</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 这里的传参是 k_cache  const scalar_t* __restrict__ k_cache,   // [num_blocks, num_kv_heads, head_size/x, block_size, x]</span></span><br><span class="line"><span class="comment">// 这里在拿到每组 k_vecs 后，直接计算了点乘（并且考虑了 mask）</span></span><br><span class="line"><span class="comment">// 输出存储在 logits</span></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> WARP_SIZE 32</span></span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line">  <span class="type">float</span> qk_max = -FLT_MAX;</span><br><span class="line"><span class="comment">// 处理每个 block table 内的 key</span></span><br><span class="line">  <span class="type">const</span> <span class="type">int</span>* block_table = block_tables + seq_idx * max_num_blocks_per_seq;</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> block_idx = start_block_idx + warp_idx; block_idx &lt; end_block_idx; block_idx += NUM_WARPS) &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int64_t</span> physical_block_number = <span class="built_in">static_cast</span>&lt;<span class="type">int64_t</span>&gt;(block_table[block_idx]);</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 为了多线程加速，将其拆分为多个块同时处理。可以理解为这里是并行？</span></span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_TOKENS_PER_THREAD_GROUP; i++) &#123;</span><br><span class="line">      <span class="type">const</span> <span class="type">int</span> physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;</span><br><span class="line">      <span class="type">const</span> <span class="type">int</span> token_idx = block_idx * BLOCK_SIZE + physical_block_offset;</span><br><span class="line">      K_vec k_vecs[NUM_VECS_PER_THREAD];</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 获取到 block 中 所有的 k 值</span></span><br><span class="line">      <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; NUM_VECS_PER_THREAD; j++) &#123;</span><br><span class="line">        <span class="type">const</span> <span class="type">scalar_t</span>* k_ptr = k_cache + physical_block_number * kv_block_stride</span><br><span class="line">                                        + kv_head_idx * kv_head_stride</span><br><span class="line">                                        + physical_block_offset * x;</span><br><span class="line">        <span class="type">const</span> <span class="type">int</span> vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;</span><br><span class="line">        <span class="type">const</span> <span class="type">int</span> offset1 = (vec_idx * VEC_SIZE) / x;</span><br><span class="line">        <span class="type">const</span> <span class="type">int</span> offset2 = (vec_idx * VEC_SIZE) % x;</span><br><span class="line">        k_vecs[j] = *<span class="built_in">reinterpret_cast</span>&lt;<span class="type">const</span> K_vec*&gt;(k_ptr + offset1 * BLOCK_SIZE * x + offset2);</span><br><span class="line">      &#125;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 计算点积</span></span><br><span class="line">      <span class="type">float</span> qk = scale * Qk_dot&lt;<span class="type">scalar_t</span>, THREAD_GROUP_SIZE&gt;::<span class="built_in">dot</span>(q_vecs[thread_group_offset], k_vecs);</span><br><span class="line">      qk += (alibi_slope != <span class="number">0</span>) ? alibi_slope * (token_idx - context_len + <span class="number">1</span>) : <span class="number">0</span>;</span><br><span class="line"> </span><br><span class="line">      <span class="keyword">if</span> (thread_group_offset == <span class="number">0</span>) &#123;</span><br><span class="line"><span class="comment">// 考虑 mask</span></span><br><span class="line">        <span class="type">const</span> <span class="type">bool</span> mask = token_idx &gt;= context_len;</span><br><span class="line">        logits[token_idx - start_token_idx] = mask ? <span class="number">0.f</span> : qk;</span><br><span class="line"><span class="comment">// 获取最大值，以计算 softmax</span></span><br><span class="line">        qk_max = mask ? qk_max : <span class="built_in">fmaxf</span>(qk_max, qk);</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br></pre></td></tr></table></figure>
-<p>计算 softmax</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 获取所有 warp 中的最大值</span></span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> mask = WARP_SIZE / <span class="number">2</span>; mask &gt;= THREAD_GROUP_SIZE; mask /= <span class="number">2</span>) &#123;</span><br><span class="line">    qk_max = <span class="built_in">fmaxf</span>(qk_max, __shfl_xor_sync(<span class="built_in">uint32_t</span>(<span class="number">-1</span>), qk_max, mask));</span><br><span class="line">  &#125;</span><br><span class="line"><span class="comment">// 放到共享存储中，只需要第一个线程放置即可</span></span><br><span class="line">  <span class="keyword">if</span> (lane == <span class="number">0</span>) &#123;</span><br><span class="line">    red_smem[warp_idx] = qk_max;</span><br><span class="line">  &#125;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 获取所有 block 中的最大值</span></span><br><span class="line">  qk_max = lane &lt; NUM_WARPS ? red_smem[lane] : -FLT_MAX;</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> mask = NUM_WARPS / <span class="number">2</span>; mask &gt;= <span class="number">1</span>; mask /= <span class="number">2</span>) &#123;</span><br><span class="line">    qk_max = <span class="built_in">fmaxf</span>(qk_max, __shfl_xor_sync(<span class="built_in">uint32_t</span>(<span class="number">-1</span>), qk_max, mask));</span><br><span class="line">  &#125;</span><br><span class="line"><span class="comment">// 将最大值广播至所有线程</span></span><br><span class="line">  qk_max = __shfl_sync(<span class="built_in">uint32_t</span>(<span class="number">-1</span>), qk_max, <span class="number">0</span>);</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 计算每一项的 exp，exp_sum 是分母，logits[i] 是分子</span></span><br><span class="line">  <span class="type">float</span> exp_sum = <span class="number">0.f</span>;</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = thread_idx; i &lt; num_tokens; i += NUM_THREADS) &#123;</span><br><span class="line">    <span class="type">float</span> val = __expf(logits[i] - qk_max);</span><br><span class="line">    logits[i] = val;</span><br><span class="line">    exp_sum += val;</span><br><span class="line">  &#125;</span><br><span class="line">  exp_sum = <span class="built_in">block_sum</span>&lt;NUM_WARPS&gt;(&amp;red_smem[NUM_WARPS], exp_sum);</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 用 1 / 分母，便于直接乘上该值，得到 softmax 后的结果</span></span><br><span class="line">  <span class="type">const</span> <span class="type">float</span> inv_sum = __fdividef(<span class="number">1.f</span>, exp_sum + <span class="number">1e-6f</span>);</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = thread_idx; i &lt; num_tokens; i += NUM_THREADS) &#123;</span><br><span class="line">    logits[i] *= inv_sum;</span><br><span class="line">  &#125;</span><br></pre></td></tr></table></figure>
-<p>取 v 和第二个矩阵乘法</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 假设为 BLOCK_SIZE</span></span><br><span class="line">  <span class="keyword">constexpr</span> <span class="type">int</span> V_VEC_SIZE = <span class="built_in">MIN</span>(<span class="number">16</span> / <span class="built_in">sizeof</span>(<span class="type">scalar_t</span>), BLOCK_SIZE);</span><br><span class="line">  <span class="keyword">using</span> V_vec = <span class="keyword">typename</span> Vec&lt;<span class="type">scalar_t</span>, V_VEC_SIZE&gt;::Type;</span><br><span class="line">  <span class="keyword">using</span> L_vec = <span class="keyword">typename</span> Vec&lt;<span class="type">scalar_t</span>, V_VEC_SIZE&gt;::Type;</span><br><span class="line">  <span class="keyword">using</span> Float_L_vec = <span class="keyword">typename</span> FloatVec&lt;L_vec&gt;::Type;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 所以这里可以为 1</span></span><br><span class="line">  <span class="keyword">constexpr</span> <span class="type">int</span> NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;</span><br><span class="line"><span class="comment">// 这里就是 WARP_SIZE</span></span><br><span class="line">  <span class="keyword">constexpr</span> <span class="type">int</span> NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;</span><br><span class="line"><span class="comment">// 拆分 HEAD_SIZE 到每个线程上</span></span><br><span class="line">  <span class="keyword">constexpr</span> <span class="type">int</span> NUM_ROWS_PER_THREAD = <span class="built_in">DIVIDE_ROUND_UP</span>(HEAD_SIZE, NUM_ROWS_PER_ITER);</span><br><span class="line"> </span><br><span class="line"><span class="comment">// accs 是最后计算的结果，会赋值给返回变量。</span></span><br><span class="line"><span class="comment">// 这里用一维实现，那么其大小应该 &gt;= n_seq * n_head * head_size</span></span><br><span class="line"><span class="comment">// 为什么可以大于？因为有 block，允许一定的冗余</span></span><br><span class="line"><span class="comment">// 为什么是一维？n_seq 被整合成了一个，n_head * head_size 被拆分到了各个线程中（简单理解为每个线程中都有这么一个数组，所以实际是两个维度），最后需要拆汇总？</span></span><br><span class="line">  <span class="type">float</span> accs[NUM_ROWS_PER_THREAD];</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_ROWS_PER_THREAD; i++) &#123;</span><br><span class="line">    accs[i] = <span class="number">0.f</span>;</span><br><span class="line">  &#125;</span><br><span class="line"> </span><br><span class="line">  <span class="type">scalar_t</span> zero_value;</span><br><span class="line">  <span class="built_in">zero</span>(zero_value);</span><br><span class="line"><span class="comment">// 同上，取每个 block 中的 v 值，以及取出 q*k 的结果 logits</span></span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> block_idx = start_block_idx + warp_idx; block_idx &lt; end_block_idx; block_idx += NUM_WARPS) &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int64_t</span> physical_block_number = <span class="built_in">static_cast</span>&lt;<span class="type">int64_t</span>&gt;(block_table[block_idx]);</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> token_idx = block_idx * BLOCK_SIZE + physical_block_offset;</span><br><span class="line">    L_vec logits_vec;</span><br><span class="line"><span class="comment">// 取出每个句子的 logits</span></span><br><span class="line">    <span class="built_in">from_float</span>(logits_vec, *<span class="built_in">reinterpret_cast</span>&lt;Float_L_vec*&gt;(logits + token_idx - start_token_idx));</span><br><span class="line"><span class="comment">// 拿到 v 的指针，根据指针取值</span></span><br><span class="line">    <span class="type">const</span> <span class="type">scalar_t</span>* v_ptr = v_cache + physical_block_number * kv_block_stride</span><br><span class="line">                                    + kv_head_idx * kv_head_stride;</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_ROWS_PER_THREAD; i++) &#123;</span><br><span class="line">      <span class="type">const</span> <span class="type">int</span> row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;</span><br><span class="line">      <span class="keyword">if</span> (row_idx &lt; HEAD_SIZE) &#123;</span><br><span class="line">        <span class="type">const</span> <span class="type">int</span> offset = row_idx * BLOCK_SIZE + physical_block_offset;</span><br><span class="line"><span class="comment">// 拿到 v 的值</span></span><br><span class="line">        V_vec v_vec = *<span class="built_in">reinterpret_cast</span>&lt;<span class="type">const</span> V_vec*&gt;(v_ptr + offset);</span><br><span class="line">        <span class="keyword">if</span> (block_idx == num_context_blocks - <span class="number">1</span>) &#123;</span><br><span class="line">          <span class="type">scalar_t</span>* v_vec_ptr = <span class="built_in">reinterpret_cast</span>&lt;<span class="type">scalar_t</span>*&gt;(&amp;v_vec);</span><br><span class="line">          <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; V_VEC_SIZE; j++) &#123;</span><br><span class="line"><span class="comment">// 避免 nan 值，置 0 处理</span></span><br><span class="line">            v_vec_ptr[j] = token_idx + j &lt; context_len ? v_vec_ptr[j] : zero_value;</span><br><span class="line">          &#125;</span><br><span class="line">        &#125;</span><br><span class="line"><span class="comment">// 计算第二个矩阵乘法</span></span><br><span class="line">        accs[i] += <span class="built_in">dot</span>(logits_vec, v_vec);</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 每个线程进行求和，汇总至 accs</span></span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_ROWS_PER_THREAD; i++) &#123;</span><br><span class="line">    <span class="type">float</span> acc = accs[i];</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> mask = NUM_V_VECS_PER_ROW / <span class="number">2</span>; mask &gt;= <span class="number">1</span>; mask /= <span class="number">2</span>) &#123;</span><br><span class="line">      acc += __shfl_xor_sync(<span class="built_in">uint32_t</span>(<span class="number">-1</span>), acc, mask);</span><br><span class="line">    &#125;</span><br><span class="line">    accs[i] = acc;</span><br><span class="line">  &#125;</span><br><span class="line"> </span><br><span class="line">... # 中间一些处理逻辑</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 将 accs 的结果，还原至返回 out 中（out_ptr 是它的指针）</span></span><br><span class="line"><span class="meta">#<span class="keyword">pragma</span> unroll</span></span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_ROWS_PER_THREAD; i++) &#123;</span><br><span class="line">      <span class="type">const</span> <span class="type">int</span> row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;</span><br><span class="line">      <span class="keyword">if</span> (row_idx &lt; HEAD_SIZE &amp;&amp; lane % NUM_V_VECS_PER_ROW == <span class="number">0</span>) &#123;</span><br><span class="line">        <span class="built_in">from_float</span>(*(out_ptr + row_idx), accs[i]);</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br></pre></td></tr></table></figure>
-<h2 id="next-step">Next Step</h2>
-<p>对于目前的 LLM 来说，一种广泛应用的场景会固定若干个 System
-Message，相当于 Prompt 是重复的。但每次生成，都需要重新计算每个 System
-Message 的 kv cache。因此，这一步是可以优化的。vLLM
-在论文中提到了这一点，但目前 vLLM 尚未实现。</p>
-<h3 id="system-message-预生成-kv-cache">System Message 预生成 kv
-cache</h3>
-<p>可以先给定若干 System Message，计算好 kv
-cache，以字典的形式存储。在请求时，带上标识符，在字典中找到对应的 kv
-cache，在生成时直接拼上，而不计算这一部分的 kv cache。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/vllm_pageattention/annimation2.gif" /></p>
-<h3 id="策略赋值更新-kv-cache">策略赋值更新 kv cache</h3>
-<p>以工具调用为例，可以预先生成工具调用结果的所有 token 的 kv
-cache，加速生成。</p>
-<ol type="1">
-<li>在实现 multi token attention 的基础上</li>
-<li>调度处理</li>
-</ol>
-<p>对于这两种都依赖于 multi token attention，而这一个 vLLM 是直接调用
-xformers 的 xops.memory_efficient_attention_forward
-实现的。这种方式当前只能在第一次生成时使用，没法拼接？</p>
+<h3 id="detail"><strong>Detail</strong></h3>
 <ul>
-<li>比如总共 10 个 prompt token，现在一次性生成 10 个 token 的 kv
-cache。期望分两次各生成前 5 个和后 5 个的 token，将 kv cache
-拼起来，得到的结果不一致</li>
+<li>When an LLM outputs information <strong>it knows to be
+false</strong>, correctly solves math or programming problems?</li>
+<li>Or begs the user not to shut it down, is it simply
+<strong>regurgitating</strong> (or splicing together) passages from the
+<strong>training set</strong>?</li>
+<li>Or is it combining its stored knowledge in <strong>creative
+ways</strong> and building on a detailed world model?</li>
 </ul>
-<p>相关文章：<a href="#">Post not found: PyTorch/pytorch_cpp_extension.md</a></p>
-<p>xops.memory_efficient_attention_forward 学习</p>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>NLP</tag>
-        <tag>Attention</tag>
-        <tag>vLLM</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>产品评论数据情感分析：从文本评论预处理到模型构建</title>
-    <url>/hexo_blog/2018/08/18/%E4%BA%A7%E5%93%81%E8%AF%84%E8%AE%BA%E6%95%B0%E6%8D%AE%E6%83%85%E6%84%9F%E5%88%86%E6%9E%90/</url>
-    <content><![CDATA[<p>NLP处理问题的框图（知识点）。To Be Continue！</p>
-<span id="more"></span>
-<h1 id="产品评论数据情感分析">产品评论数据情感分析</h1>
-<h2 id="数据获取">数据获取</h2>
-<h2 id="文本评论预处理">文本评论预处理</h2>
-<h4 id="编码问题可选">编码问题（可选）</h4>
-<h4 id="文本去重">文本去重</h4>
-<h6 id="基本解释">基本解释</h6>
-<p>大量重复且无实际意义的数据，需要删除</p>
-<h6 id="算法">算法</h6>
-<p>编辑距离去重
-计算两条语料的编辑距离，与阈值（提前设定好的一个值）进行判断，小于阈值就进行去重
-优点：有效去除了接近重复或完全重复的数据
-缺点：错删语义相近的语料，两句话虽然相似但并不需要删除。由于阈值设定的原因可能会错删。
-Simhash算法去重 k-shingle算法(用的不多) Minhash算法</p>
-<h6 id="选用的方法">选用的方法</h6>
-<p>最简单粗暴：完全重复的语料进行剔除。见 <code>programmer_2</code></p>
-<h4 id="机械压缩去词">机械压缩去词</h4>
-<h6 id="基本解释-1">基本解释</h6>
-<p>经过上面的处理之后，还有这样一部分数据并没有去除。如"好的好的好的好的好的好的好的好的好的"#
-我们称之为连续重复语料，最常见的较长的无意义语料。</p>
-<h6 id="处理方法">处理方法</h6>
-<p>在某些情况下，重复词带有一定的情感倾向，如果直接进行处理，会影响情感倾向的判断。
-所以这里需要对语料进行机械压缩去词处理，去除一些连续累赘的表达（不影响情感）。比如上面的句子，压缩为"好的"
-一般情况下，这种词汇常出现在开头或者结尾。语料中间的重复词汇可能具有实际意义比如"滔滔不绝"中的滔滔是具有实际意义的。因此只对开头与结尾的连续重复词进行操作。</p>
-<h6 id="设定规则">设定规则</h6>
-<p>考虑因素：词法结构、文字表达特点 七个规则</p>
-<h4 id="短句删除">短句删除</h4>
-<h6
-id="字数越少表达的意思就少所以需要过滤掉过短的评论文本数据">字数越少表达的意思就少，所以需要过滤掉过短的评论文本数据。</h6>
-<h2 id="文本评论分词">文本评论分词</h2>
-<h4 id="分词介绍">分词介绍</h4>
-<h6 id="python库jieba">python库：jieba</h6>
-<h6 id="引入停用词">引入停用词</h6>
-<h6 id="特征处理">特征处理</h6>
-<h6 id="中文分词方法">中文分词方法</h6>
-<p>基于规则的分词方法 基于统计的分词方法 基于语义的分词方法
-基于理解的分词方法</p>
-<h2 id="模型构建">模型构建</h2>
-<h4 id="情感倾向性模型">情感倾向性模型</h4>
-<h6 id="训练生成词向量">训练生成词向量</h6>
-<p>One-hot# Representation Distributed# Representation word2vec</p>
-<h6 id="人工标注与映射">人工标注与映射</h6>
-<p>snownlp</p>
-<h6 id="训练模型">训练模型</h6>
-<p>栈式自编码网络 基本介绍</p>
-<h4 id="语义网络分析">语义网络分析</h4>
-<h6 id="语义网络">语义网络</h6>
-<p>介绍 步骤 优势</p>
-<h4 id="模型选择">模型选择</h4>
-<h6 id="lda模型的主题分析">LDA模型的主题分析</h6>
-<p>模型介绍 Bag Of Words（BOW，词袋模型） 模型估计 实现过程
-拓展：ILDA</p>
-<h6 id="vsm与tf-idf">VSM与TF-IDF</h6>
-<h6 id="svd与lsa">SVD与LSA</h6>
-<h6 id="plsa与lda">PLSA与LDA</h6>
-<h4 id="构建模型">构建模型</h4>
-<h6 id="svm-rbf-pca">SVM (RBF) + PCA</h6>
-<h6 id="subtopic">Subtopic</h6>
-<h2 id="模型评价">模型评价</h2>
-<h4 id="应用层次分析法">应用层次分析法</h4>
-<h4 id="模糊综合评价">模糊综合评价</h4>
-]]></content>
-      <categories>
-        <category>NLP</category>
-      </categories>
-      <tags>
-        <tag>数据分析</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>待整理的脚本（持续更新ing）</title>
-    <url>/hexo_blog/2023/11/26/%E5%BE%85%E6%95%B4%E7%90%86%E8%84%9A%E6%9C%AC/</url>
-    <content><![CDATA[<p>记录一些常用/重要，但容易忘记的命令</p>
-<span id="more"></span>
-<h3 id="git-统计不同版本的文件夹代码行数">Git
-统计不同版本的文件夹代码行数</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git checkout &lt;commit_id&gt;</span><br><span class="line">find -name <span class="string">&quot;*.py&quot;</span> | xargs <span class="built_in">cat</span> | <span class="built_in">wc</span> -l</span><br></pre></td></tr></table></figure>
-<h3 id="git-撤销-commit">Git 撤销 commit</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git revert &lt;commit_id&gt;</span><br></pre></td></tr></table></figure>
-<h3 id="git-撤销单个文件的修改">Git 撤销单个文件的修改</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git checkout HEAD -- &lt;文件路径&gt;</span><br></pre></td></tr></table></figure>
-<h3 id="git-暂存">Git 暂存</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git stash</span><br><span class="line">git stash pop</span><br></pre></td></tr></table></figure>
-<h3 id="grep-搜索文本并显示文件名和行号">Grep
-搜索文本并显示文件名和行号</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">grep -Hn <span class="string">&quot;Hello World!&quot;</span> *</span><br></pre></td></tr></table></figure>
-<h3 id="grep-搜索不包含某个字符串的文本并显示文件名和行号">Grep
-搜索不包含某个字符串的文本并显示文件名和行号</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">grep -Hn <span class="string">&quot;Hello World!&quot;</span> * | grep -v <span class="string">&quot;XXX&quot;</span></span><br></pre></td></tr></table></figure>
-<h3 id="python-注册全局变量以在所有代码中使用">Python
-注册全局变量以在所有代码中使用</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="built_in">globals</span>()[<span class="string">&quot;seed&quot;</span>] = <span class="number">42</span></span><br></pre></td></tr></table></figure>
-<h3 id="python-动态修改源码-monkey-patching">Python 动态修改源码 (Monkey
-Patching)</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在源码中</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">A</span>:</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self</span>):</span><br><span class="line">        ...</span><br><span class="line"></span><br><span class="line"><span class="comment"># 在项目中</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">new_forward</span>(<span class="params">self</span>):</span><br><span class="line">    ...</span><br><span class="line">A.forward = new_forward</span><br></pre></td></tr></table></figure>
-<h3 id="wget-下载文件保存为时间戳">wget 下载文件，保存为时间戳</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget -O `<span class="built_in">date</span> +%s`.jpg https://www.baidu.com/img/bd_logo1.png</span><br></pre></td></tr></table></figure>
-<h3 id="nginx-端口转发">Nginx 端口转发</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">server &#123;</span><br><span class="line">    listen 4747;</span><br><span class="line">    server_name localhost;</span><br><span class="line">    proxy_pass http://x.x.x.x:4747;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h3 id="linux-统计当前路径下的文件个数">Linux
-统计当前路径下的文件个数</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">ls</span> -l | grep <span class="string">&quot;^-&quot;</span> | <span class="built_in">wc</span> -l</span><br></pre></td></tr></table></figure>
-<h3 id="kill-python-相关进程">kill Python 相关进程</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">ps -ef | grep python | grep -v grep | awk <span class="string">&#x27;&#123;print $2&#125;&#x27;</span> | xargs <span class="built_in">kill</span> -9</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Linux</category>
-      </categories>
-      <tags>
-        <tag>Note</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>年度推荐软件Tailscale：连接团队设备和开发环境的利器</title>
-    <url>/hexo_blog/2023/04/03/%E5%B9%B4%E5%BA%A6%E6%8E%A8%E8%8D%90%E8%BD%AF%E4%BB%B6Tailscale/</url>
-    <content><![CDATA[<p>本文介绍了 Tailscale
-这款软件可以轻松架设异地组网，解决调试难题、实现流量转发和开放服务等多个需求。</p>
-<span id="more"></span>
-<p><a href="https://tailscale.com/">Tailscale</a> connects your team's
-devices and development environments for easy access to remote
-resources. （摘自官网）</p>
-<p>于笔者而言，Tailscale 能够用一个软件轻松的解决三大需求。</p>
+<h2 id="how">How</h2>
 <ul>
-<li>异地组网：软件主打功能。就是将物理位置不同的机器通过该软件，能够使它们处于同一网络环境下，方便远程调试。</li>
-<li>流量转发：A机器在A地，可以借助B机器在B地的网络进行上网。</li>
-<li>开放服务：A机器自建Web服务，不仅可以在组网环境下使用，甚至官方提供了公网的链接（https），即可以不在组网环境下也能够访问。</li>
+<li><p>自顶向下思考，在给定某个输入的情况下，为什么模型会有这种输出？</p>
+<ul>
+<li><strong>训练数据</strong> +模型+优化方法</li>
+</ul></li>
+<li><p>模型是记住了数据还是理解了数据？</p>
+<ul>
+<li><a href="https://arxiv.org/abs/1906.01820">Model could be
+deceptively aligned</a></li>
+</ul></li>
 </ul>
-<p>写在开头，免费版的权益如下图所示，支持20个设备，且域名不可自定义（子域名可以），速度在100-300k/s左右。在Android和iOS设备上不支持同时开启两个
-VPN，即 Clash 无法和 Tailscale 同时使用。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680514288836.png" /></p>
-<h2 id="安装">安装</h2>
-<p>首先需要在官网进行注册，值得一提的是注册方式最好选用能够在日常网络环境下能打开的账号。比如选用Github注册的话，之后所有的设备都需要使用Github登录才能组网。其目前无法在一个账号上绑定多种登录方式。</p>
-<p>官网提供了详尽的<a
-href="https://tailscale.com/download">下载</a>安装说明，移动端可以直接下载对应的App，个人电脑和服务器可以使用常规的命令和软件包的方式进行下载安装。故对于一般的机器不做介绍，本节主要介绍路由器、树莓派、群晖的安装方法。其他机型，可以查看这个<a
-href="https://pkgs.tailscale.com/stable/">地址</a>。</p>
-<p>对于Linux设备，安装步骤主要分两步：</p>
-<ol type="1">
-<li>启动服务，<code>sudo tailscale up</code></li>
-<li>加入组网，<code>tailscale ip -4</code></li>
-</ol>
-<p>对于第二步，输入完成后会有一个链接需要使用对应账号进行登录，可以将链接复制到其他设备用浏览器打开登录。在非常规机器上，首先需要确认机器的CPU架构，命令
-<code>uname -a</code>。</p>
-<h3 id="群晖">群晖</h3>
-<p>群晖的<a
-href="https://pkgs.tailscale.com/stable/#spks">下载地址</a>，需要根据系统版本以及CPU架构选择下载。下载完成后，打开群晖的<strong>套件中心</strong>，右上角的<strong>手动安装</strong>，上传安装包，按照步骤完成安装即可。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/170356.png" /></p>
-<p>安装完成后，点击对应的图标，安装对应的步骤即可启动。</p>
-<h3 id="路由器">路由器</h3>
-<p>路由器的<a
-href="https://pkgs.tailscale.com/stable/#static">下载地址</a>，根据CPU架构选择下载。并将安装包上传至路由器，根据对应的路由器安装命令进行解压安装。</p>
-<h3 id="树莓派">树莓派</h3>
-<p>树莓派的<a
-href="https://pkgs.tailscale.com/stable/#raspbian-stretch">下载地址</a>，根据型号三选一下载。解压安装。树莓派Zero的是Raspbian
-Stretch</p>
-<h2 id="异地组网">异地组网</h2>
-<p>当设备安装完成，并登录后，该设备会自动加入组网。可以在<a
-href="https://login.tailscale.com/admin/machines">设备管理</a>界面查看所有的设备，如下图所示。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/171953.png" /></p>
-<p>第一列的设备名称可以根据自己的喜好进行更改，并且可以直接作为域名使用。假设有个机器叫做AAA且用户名为root，那么可以直接
-<code>ssh root@AAA</code>进行登录。或者群晖名为AAA，那么浏览器直接打开AAA就能访问。</p>
-<p>登录的设备默认是若干天（最长180）后需要重新登录一次，可以在最后一列三个点进行编辑，对该功能进行禁用。禁用成功后，会在第一列显示
-<code>Expiry disabled</code></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/172831.png" /></p>
-<h2 id="流量转发">流量转发</h2>
-<p>本节参考<a href="https://tailscale.com/kb/1019/subnets/">文档</a></p>
-<p>正如开头的图显示的，该功能仅支持以一个设备作为流量出口。当有一个局域网的所有设备不方便接入组网时，可以通过接入该局域网的一个设备（如路由器），再访问该局域网的其他设备。如下图，<code>Subnet Router</code>相当于桥梁，承接了组网和本地组网。组网内的设备可以通过Router访问局域网内的设备，反之亦可。接下来在
-<code>Router</code>设备上操作。</p>
-<p><img src="https://tailscale.com/kb/1019/subnets/subnets.png" /></p>
-<h3 id="step-1">Step 1</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 开启端口转发。有的机器可能默认开启，无需操作。</span></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&#x27;net.ipv4.ip_forward = 1&#x27;</span> | <span class="built_in">sudo</span> <span class="built_in">tee</span> -a /etc/sysctl.d/99-tailscale.conf</span><br><span class="line"><span class="built_in">echo</span> <span class="string">&#x27;net.ipv6.conf.all.forwarding = 1&#x27;</span> | <span class="built_in">sudo</span> <span class="built_in">tee</span> -a /etc/sysctl.d/99-tailscale.conf</span><br><span class="line"><span class="built_in">sudo</span> sysctl -p /etc/sysctl.d/99-tailscale.conf</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建Subnet，192.168.x.y可以换成10.0这种。可以只写一个，x.y替换任意数字</span></span><br><span class="line"><span class="built_in">sudo</span> tailscale up --advertise-routes=192.168.x.y/24,192.168.x.z/24</span><br></pre></td></tr></table></figure>
-<h3 id="step-2">Step 2</h3>
-<p>在机器管理界面，点击对应机器的右侧三个点，再点击
-<code>Edit route settings</code>。如下图所示，两个功能均开启。若仅需实现设备访问，不需要流量转发，则可以不开启第二个。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680522033220.png" /></p>
-<h3 id="step-3">Step 3</h3>
-<p>自此，完成了桥接局域网和组网的功能。要实现流量转发，在Tailscale中叫做Exit
-Node。一方面需要在上一步开启第二个功能，此时Router就具备了作为流量出口的能力。另一方面，假设此时组网内的A设备需要借助Router的流量出口，就需要在A设备上进行操作。</p>
-<p>在Router设备上，</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建Subnet，192.168.x.y可以换成10.0这种。可以只写一个，x.y替换任意数字</span></span><br><span class="line"><span class="built_in">sudo</span> tailscale up --advertise-exit-node --advertise-routes=192.168.x.y/24</span><br></pre></td></tr></table></figure>
-<p>一般有图形化界面的机器，可以直接在软件层面完成。如Windows，此时系统的所有流量均为走这个接口。</p>
-<p><strong>Windows</strong> <img
-src="https://tailscale.com/kb/1103/exit-nodes/exit-node-windows-menu.png" /></p>
-<p><strong>Linux</strong></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">sudo</span> tailscale up --exit-node=&lt;exit-node-ip&gt;</span><br></pre></td></tr></table></figure>
-<p>该功能主要的用途有以下几点</p>
-<ul>
-<li>本节开头所提到的，通过一个Router，从而无需所有设备参与链接。</li>
-<li>流量转发，如果有一个节点在海外。那么可以以此来做跳板</li>
-<li>本文开头所提到的，移动设备上无法与其他VPN并存，那么此时通过这种方式，在Router设备上做分流。</li>
-</ul>
-<h2 id="开放服务">开放服务</h2>
-<p>截至本文完成日期，该功能还在Beta阶段，并且要求版本&gt;=1.38.3。<a
-href="https://tailscale.com/kb/1223/tailscale-funnel/">详细文档</a></p>
-<h3 id="step-1-1">Step 1</h3>
-<p>如果仅需在组网内访问，则可以跳过该步骤。</p>
-<p>在<a href="https://login.tailscale.com/admin/acls">Access
-controls</a>控制页面加入配置，新增 <code>nodeAttrs</code>和修改
-<code>groups</code>，其中
-<code>wnma3mz@github</code>替换为对应账号。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680523192944.png" />
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680523204260.png" /></p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;groups&quot;</span><span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">     <span class="attr">&quot;group:can-funnel&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">     <span class="string">&quot;wnma3mz@github&quot;</span><span class="punctuation">,</span></span><br><span class="line">     <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line"> <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">&quot;nodeAttrs&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">     <span class="punctuation">&#123;</span></span><br><span class="line">         <span class="attr">&quot;target&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="string">&quot;group:can-funnel&quot;</span><span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">         <span class="attr">&quot;attr&quot;</span><span class="punctuation">:</span>   <span class="punctuation">[</span><span class="string">&quot;funnel&quot;</span><span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">     <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line"> <span class="punctuation">]</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
-<h3 id="step-2-1">Step 2</h3>
-<p>此时，需要在A机器上开启服务。申请<a
-href="https://tailscale.com/kb/1153/enabling-https/">HTTPS证书</a>，下面的命令会提示后面需要加入什么域名。按照提示复制再执行即可。可能会有一些报错，需要根据不同情况进行更改。该功能可能会影响提供公网服务，但不影响组网服务。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale cert</span><br></pre></td></tr></table></figure>
-<p>在开放服务前有两个命令需要进行区分，<code>tailscale serve</code>和<code>tailscale funnle</code>前者是开放服务到组网环境，后者是开放到整个互联网环境。</p>
-<p>常用有两个技巧</p>
-<ul>
-<li>把已经开放服务的端口转发到其他端口，或者主域名下。比如这里可以把5000端口转发到主域名下的/api路由下，即可以通过
-<code>http://A/api</code>访问A机器的5000端口服务。<code>--bg</code>表示后台运行，默认是在前台启动，便于<code>Ctrl+C</code>关闭。</li>
-</ul>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve --<span class="built_in">bg</span> --set-path /api http://127.0.0.1:5000/</span><br></pre></td></tr></table></figure>
-<p>把 4747 端口转发到 8443 端口，即可以通过
-<code>https://A:8443</code>访问A机器的 4747 端口服务。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve https:8443 / http://localhost:4747</span><br></pre></td></tr></table></figure>
+<h3 id="influence-functions"><a
+href="https://www.jstor.org/stable/2285666">Influence Functions</a></h3>
 <ul>
-<li>部署静态服务</li>
+<li>Seeing which training sequences are <strong>highly
+influential</strong> can help separate out different hypotheses for why
+an output was <strong>generated</strong> and illuminate what sorts of
+structure are or are <strong>not generalized</strong> from training
+examples</li>
+<li><strong>分析各种泛化相关的现象</strong></li>
 </ul>
-<p>假设本地有一个<code>/data/index.html</code>文件，想要开放访问。则运行下面的命令，可以通过<code>http://A/index</code>访问A机器的<code>/data/index.html</code>文件。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve --<span class="built_in">bg</span> --set-path /index /data/index.html</span><br></pre></td></tr></table></figure>
-<p>如果<code>index.html</code>文件用了本机其他静态资源，同样需要进行挂载，比如挂载<code>/data/static</code>文件夹</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve --<span class="built_in">bg</span> --set-path /static/ /data/static</span><br></pre></td></tr></table></figure>
-<p>如果要关闭服务，可以使用下面的命令，仅关闭这一条。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve --<span class="built_in">bg</span> --set-path /api off</span><br></pre></td></tr></table></figure>
-<p>如果希望指定端口，则可以用下面命令，默认端口为 80。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 带端口开放</span></span><br><span class="line">tailscale serve --<span class="built_in">bg</span> --http 10001 --set-path /index /data/index.html</span><br><span class="line"><span class="comment"># 带端口关闭</span></span><br><span class="line">tailscale serve --<span class="built_in">bg</span> --http 10001 --set-path /index off</span><br></pre></td></tr></table></figure>
-<h3 id="step-3-1">Step 3</h3>
-<p>对外开放服务，下面的命令会直接把原本仅能在局域网内访问的服务提供到公网。实测下载速度200-300k/s，有些地方不能直接访问，可能要挂梯子。目前
-tailscale 仅支持三个端口可以dui外开放，分别是 443、8443、10000。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale funnel --<span class="built_in">bg</span> https+insecure://localhost:443</span><br><span class="line">tailscale funnel --<span class="built_in">bg</span> https+insecure://localhost:8443</span><br><span class="line">tailscale funnel --<span class="built_in">bg</span> https+insecure://localhost:10000</span><br></pre></td></tr></table></figure>
-<p>使用方法同 <code>serve</code>，换成 <code>funnel</code> 即可。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看状态的命令</span></span><br><span class="line">tailscale funnel status</span><br><span class="line"><span class="comment"># 关闭状态的命令</span></span><br><span class="line">tailscale funnel 443 off</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Software</category>
-      </categories>
-      <tags>
-        <tag>Tailscale</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>本科科研的那些事</title>
-    <url>/hexo_blog/2018/09/30/%E6%9C%AC%E7%A7%91%E7%A7%91%E7%A0%94%E7%9A%84%E9%82%A3%E4%BA%9B%E4%BA%8B/</url>
-    <content><![CDATA[<p>本科科研经历不仅对建立人脉、提升简历很重要，对发现兴趣、培养批判性思维能力也很重要。加入一个有可靠的导师和同行的合适的研究实验室，不仅可以帮助提高一个人的学术技能，还可以提供接触科学方法的机会，并帮助明确职业目标。本文仅叙说个人经历，从选择研究课题到制定研究方法，以及学习文献研究、编程和硬件等实用技能的重要性。作者还强调了在研究工作中追求科学兴趣和独立的重要性。</p>
-<span id="more"></span>
-<h2 id="本科科研的那些事">本科科研的那些事</h2>
-<p>本科阶段，以就业为目的的同学加入一个实验室其实是无所谓的，除非是需要倚重相关老师。其他无论是考/保研，还是出国留学的同学，选择并加入一个合适的实验室还是十分重要的。</p>
-<p>一个好的实验室往往会有靠谱导师和若干师兄师姐。从建立人脉网络上来看，导师可能会在你部分专业课程上帮助你打出高分，在考/保研阶段能够帮你推荐导师。师兄师姐也有庞大的校友资源，所以起码在信息资源上就比其他人更领先一筹。再退一步来说，进入实验室也可以为本科阶段的简历添上一笔。亦或是，为迷茫的同学更进一步了解读研的工作生活，了解清楚科研到底是什么，如何进行科研，所属专业内的学术圈子是如何如何的。</p>
-<p>就我个人而言，大二下时加入了一个新成立且庞大的实验室。大概工作就是从零造一台医疗仪器，所以学生来源于各个专业，机械、电子、材料、计算机等。各自专业的分工也不同。我被分配的课题是"大规模系统响应矩阵的快速分解"，简单来说就是对矩阵进行分解，只不过现在这个矩阵规模很大，在分解的时候我们还要追求速度。</p>
-<p>所以当一个问题抛出来时，在没有外界帮助的前提下，如何解决这个问题。我自己的思考与解决步骤如下：直接在知网中搜索这一长串名词，进行简单的文献阅读后，提取得到”矩阵分解“这一关键词。再分别用中英文对这一关键词进行搜索，再进行二次学习。最终要达到能够比较清楚的解释这是什么，目前的研究进展（简要了解），目前的研究方法（简要了解）等。这基本就属于论文中的
-<code>Introduce</code>部分。</p>
-<p>下一步，如何解决这个问题。就拿矩阵分解这一问题来说，搜索矩阵分解这一方法，很容易发现有很多方法，其中QR、LU、SVD是用的比较多。这时候就需要根据问题的具体需求（速度、准确度...）来选择合适的方法。选定方法后，就需要考虑是改进现有的方法（数学算法上）还是从其他角度入手。显然，在改进数学算法上难度还是摆在那里的。所以这里考虑一些其他方法，其他问题也可以以类似思路来解决。首先，这个问题我们是用计算机来解决的，那么显然可以想到两点，编程上进行加速、硬件上进行加速。硬件上加速简单来说就是用最好的硬件平台（计算资源），当然这样基本不具任何创新性。还可以使用诸如分布式计算这种方法来加速，这样问题就可以转换为如何将矩阵分解应用到分布式系统中，如何加速，这样问题就成功转换了。另一点，在程序上进行加速，这里就需要引入几个简单的名词：并发、并行、异步、流式计算。当然还有一些其他技巧就不展开了。这里我选用的是并行计算，那么新的问题就是如何进行矩阵分解的并行计算，是使用何种框架何种语言等等。这大概是我整个的一个思考思路，当然很多思路也可以从其他文献的阅读中进行获取。还有一个点就是，可以多种方法结合，比如这里可以做分布并行计算，当然这一点其实创新性其实也比较差，毕竟都是现有方法，除非有一些不同与这两个方法的点。</p>
-<p>所以，不仅可以学到这种思考方式，还在其中可以学习到其他技巧，比如如何搜索并下载文献，如何阅读文献等。再说回自己，由于实验室历史沉淀不足，所以很多东西都需要从零开始并自己动手。所以需要从零选购计算机配件，组装，搭建环境，独立编程等。以上可能老师也没时间来指导，基本上也都是给一个大方向没有比较具体的解决方案。所以基本都是自己或跟小伙伴一起完成。从中收获到的自然是学习到了独当一面的能力，自己动手能力又提高了；但缺点也很明显，学习时间成本是巨大的，并且老师可能会让你跑腿做其他事情。所以在这里建议，这种实验室还是慎入，当然如果为了锻炼自己一下，也可以试试，毕竟最后成就感也是十足的。我自己最后大概就是将QR、LU、SVD这三种算法进行了一个复现，SVD是用CUDA完成的。<a
-href="https://github.com/wnma3mz/Matrix-factorization">链接</a>。基于种种原因，后续并没有进行下去，即没有成果的产出，不过学到了东西才是最重要的（23333）。</p>
-<p>再谈本学院科研训练，基本也是类似于加入一个实验室这种。这里不同于上面的点在于，老师在这里基本会给出一个比较成熟的方案给予复现，所谓材料约等于搬砖（炒菜同理）。相反在这里学习到的东西其实并不算多，只是实验（非计算机）的操作步骤，仪器的使用，说白了技术性真的不算高。但主要还是方案的提出与方法的改进这种。最后的成果也是一篇SCI，其余倒真的没什么好说。</p>
-<p>就像文章中一开始说的那样，提前进入实验室能够让我认清自己是否能够做科研工作，是否喜欢。就拿科研训练这个来说，通过这个基本也认清了学科的特性，确实也没啥意思，所以算是自主劝退了。关于材料学科的特性以及各种劝退文，我在这不是很想介绍，具体可见知乎。个人还是秉持着理性劝退的态度，认清自己的追求再做决定。再谈那个实验室，就工作内容而言还是可以接受的，但是由于缺乏一定的动力（比如各种都需要自己独立完成，方向上不是自己将来想做的，老师指导时间少），故也选择了退出。不过总的来说，这两次经历确实收获颇多，着实为之后的很多工作打好了一个良好的基础，自己的思考角度也更加全面与丰富了（因人而异）。同时也认清一点就是，很多看似对自己无用的东西，其实后续都有很大的帮助。比如写博客这件小事，无形中对我论文写作提供了一个基础（能够有逻辑、清晰的阐述某件事）；再比如大一时被强制学习的
-<code>LaTex</code>，也十分有效（当然本来用word也行）；再比如SVD矩阵分解，可能当时没有想到一年后的自己还会继续做这个相关内容（本文写作于2019年9月）等。在大部分事情上，还是不要太功利，以兴趣为导向，说不定将来某一天会给予你巨大的帮助。</p>
-]]></content>
-      <categories>
-        <category>Life</category>
-      </categories>
-  </entry>
-  <entry>
-    <title>杂记</title>
-    <url>/hexo_blog/2018/05/30/%E6%9D%82%E8%AE%B0/</url>
-    <content><![CDATA[<p>SimHash算法是一种用于计算文档相似度的算法，可以通过计算文本的SimHash值来比较文档间的汉明距离，从而比较文档间的相似度。本文介绍了SimHash算法的基本原理和实现步骤，并介绍了中文分词方法和不同类型的分词算法。同时，还对损失函数、汉明距离、hash函数、频率与频度、互现信息等关键概念进行了详细解释，有助于读者深入了解SimHash算法背后的技术原理。</p>
-<span id="more"></span>
-<h2 id="simhash算法原理">SimHash算法原理</h2>
-<p><code>SimHash</code>是用来计算文档之间的相似度。通过该算法计算每篇文档的
-<code>SimHash</code>值，通过计算得到的二进制来比较文档间的<a
-href="#hanming">汉明距离</a>，进而来比较文档间的相似度。</p>
-<p>步骤：</p>
-<ol type="1">
-<li><p>分词</p>
-<p>对一段文本进行切割，得到有效的特征向量（词）。为每个特征向量设置对应的权值（1-5），权值越高则代表这个特征向量在文本中越重要。</p></li>
-<li><p>hash</p>
-<p>利用 <code>hash</code>函数计算各个特征向量的 <code>hash</code>值。<a
-href="#hash">hash函数</a></p></li>
-<li><p>加权</p>
-<p>在 <code>hash</code>之后，对每个特征向量进行加权，即<strong>W = Hash
-* Weight</strong></p></li>
-<li><p>合并</p>
-<p>在对每个特征向量加权后，再累加，变成一个序列串。</p></li>
-<li><p>降维</p>
-<p>由于最后合并的数字可能过大（过小），设定一个阈值，大于某阈值的设置为1，小于某阈值的设置为0。有很多降维处理的操作，这里只是简单举个例子</p></li>
-</ol>
-<p>举例：</p>
-<p>文本：人生苦短，我用Python</p>
-<ol type="1">
-<li><p>分词。</p></li>
-<li><p>hash。</p></li>
-<li><p>加权。遇到0就变为负数</p></li>
-<li><p>合并。对应位加和。</p></li>
-<li><p>降维。大于0的设置为1，否则为0。</p>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">分词	  hash				加权 			合并			 降维</span><br><span class="line"></span><br><span class="line">生(5)  人生(&quot;10101&quot;)	  	5 -5 5 -5 5	  5+5+4+3-3-3-2=9 	1</span><br><span class="line">ython(5)  Python(&quot;11101&quot;)	5 5 5 -5 5	   -5+5-4-3-3-3-2=-15	0</span><br><span class="line">(4) 	 苦(&quot;10110&quot;)		  4 -4 4 4 -4	 5+5+4+3+3-3-2=15	  1</span><br><span class="line">(3)	 短(&quot;00101&quot;)		 3 -3 3 3 -3	 -5-5+4+3-3-3+2=-7	 0</span><br><span class="line">(3)	 我(&quot;00001&quot;)		 -3 -3 3 -3 3	 5+5-4-3+3+3-2=7	1</span><br><span class="line">(2)  	 用(&quot;00010&quot;)		 -2 -2 -2 2 -2</span><br></pre></td></tr></table></figure></li>
-</ol>
-<h2 id="中文分词方法">中文分词方法</h2>
-<h3 id="基于规则的分词方法">基于规则的分词方法</h3>
-<p>机械分词、基于字典。按照一定的策略在一个词典中的词条进行匹配，若找到某词条则匹配成功。</p>
-<p>三要素：</p>
+<p>研究是基于预训练模型，而不是微调后的模型</p>
+<h2 id="findings">Findings</h2>
 <ul>
-<li><p>分词词典</p></li>
-<li><p>文本扫描顺序</p>
+<li><p>Typical model behaviors do <strong>not</strong> result from
+<strong>direct memorization</strong> of a handful of sequences</p></li>
+<li><p>Larger models consistently generalize at a <strong>more abstract
+level</strong> than smaller models</p>
 <ul>
-<li>正向扫描</li>
-<li>逆向扫描</li>
-<li>双向扫描</li>
+<li>role-playing behavior, programming, mathematical reasoning, and
+cross-lingual generalization</li>
 </ul></li>
-<li><p>匹配原则</p>
+<li><p>Role-playing behavior is influenced primarily by examples or
+descriptions of similar behaviors in the training set</p>
 <ul>
-<li><p>最大匹配（MM）</p>
-<p>假定词典中的最长词条所含汉字个数为i，则被处理文本中前i个字符作为匹配字段，查找词典。如果词典中含有这样i个汉字个数的词，则匹配成功，并且切分出这个词；如果找不到就匹配失败，将匹配字段去掉最后一个汉字，继续匹配，直到匹配成功为止。统计方法表明，错误率为1/169。</p></li>
-<li><p>最小匹配（逆向最大匹配，RMM）。与MM方法相同，只不过是从文本末尾开始，匹配失败就去掉前一个汉字。错误率为1/245。</p></li>
-<li><p>逐词匹配。将词典中的词由长到短逐词遍历，直到文本切分完成。</p></li>
-<li><p>最佳匹配（OM）</p>
-<p>在词典中按词频的大小顺序排列词条，缩短检索时间，降低时间复杂度，加快分词速度。不算是一种分词方法，只是一种对分词词典的一种组织方式。OM
-法的分词词典每条词的前面必须有指明长度的数据项，所以其空间复杂度有所增加，对提高分词精度没有影响，分词处理的时间复杂度有所降低。</p></li>
+<li>suggesting that the behaviors result from <strong>imitation</strong>
+rather than sophisticated planning</li>
 </ul></li>
 </ul>
-<h3 id="基于统计的分词方法">基于统计的分词方法</h3>
-<p>基本思想：词是稳定的组合。</p>
-<p>上下文中，相邻的字同时出现的次数越多，就越有可能构成一个词。</p>
-<p>字与字相邻出现的概率或<a
-href="#frequency">频率</a>反应出词的可信度。</p>
-<p>无字典分词：统计文本中相邻字的组合的<a
-href="#frequency">频度</a>，计算它们之间的互现信息。</p>
-<p>该方法所应用的主要的统计模型有：N
-元文法模型（N-gram）、隐马尔可夫模型（Hiden Markov
-Model，HMM）、最大熵模型（ME）、条件随机场模型（Conditional Random
-Fields，CRF）等。</p>
-<p>在实际应用中此类分词算法一般是将其与基于词典的分词方法结合起来，既发挥匹配分词切分速度快、效率高的特点，又利用了无词典分词结合上下文识别生词、自动消除歧义的优点。</p>
-<h3 id="基于语义的分词方法">基于语义的分词方法</h3>
-<p>语义分析。对自然语言自身的语言信息进行更多的处理。</p>
-<ul>
-<li><p>扩充转移网络法</p>
-<p><a
-href="https://baike.baidu.com/item/%E6%9C%89%E9%99%90%E7%8A%B6%E6%80%81%E6%9C%BA/2081914?fr=aladdin">有限状态机</a>。有限状态机只能识别正则语言，对有限状态机作的第一次扩充使其具有递归能力，形成递归转移网络
-（RTN）。在RTN
-中，弧线上的标志不仅可以是终极符（语言中的单词）或非终极符（词类），还可以调用另外的子网络名字分非终极符（如字或字串的成词条件）。这样，计算机在
-运行某个子网络时，就可以调用另外的子网络，还可以递归调用。词法扩充转移网络的使用，
-使分词处理和语言理解的句法处理阶段交互成为可能，并且有效地解决了汉语分词的歧义。</p></li>
-<li><p>知识分词语义分析法</p></li>
-<li><p>邻接约束法</p></li>
-<li><p>综合匹配法</p></li>
-<li><p>后缀分词法</p></li>
-<li><p>特征词库法</p></li>
-<li><p>矩阵约束法</p>
-<p>先建立一个语法约束矩阵和一个语义约束矩阵，
-其中元素分别表明具有某词性的词和具有另一词性的词相邻是否符合语法规则，
-属于某语义类的词和属于另一词义类的词相邻是否符合逻辑，机器在切分时以之约束分词结果。</p></li>
-<li><p>语法分析法</p></li>
-<li><p>……</p></li>
-</ul>
-<h3 id="基于理解的分词方法">基于理解的分词方法</h3>
-<p>通过让计算机模拟人对句子的理解，达到识别词的效果。这种分词方法需要使用大量的语言知识和信息。</p>
-<p>基本思想：在分词的同时进行句法、语义分析，利用句法信息和语义信息来处理歧义现象。</p>
-<ul>
-<li>分词子系统</li>
-<li>句法语义子系统</li>
-<li>总控部分</li>
-</ul>
-<p>在总控部分的协调下，分词子系统可以获得有关词、句子等的句法和语义信息来对分词歧义进行判断，即它模拟了人对句子的理解过程。</p>
-<p>主要的方法：</p>
+<h3 id="notes">Notes</h3>
 <ul>
-<li><p>专家系统分词法</p>
-<p>从专家系统角度把分词的知识（包括常识性分词知识与消除歧义切分的启发性知识即歧义切分规则）从实现分词过程的推理机中独立出来，使知识库的维护与推理机的实现互不干扰，从而使知识库易于维护和管理。它还具有发现交集歧义字段和多义组合歧义字段的能力和一定的自学习功能。</p></li>
-<li><p>神经网络分词法</p>
-<p>该方法是模拟人脑并行，分布处理和建立数值计算模型工作的。它将分词知识所分散隐式的方法存入神经网络内部，通过自学习和训练修改内部权值，以达到正确的分词结果，最后给出神经网络自动分词结果，如使用
-LSTM、GRU 等神经网络模型等。</p></li>
-<li><p>神经网络专家系统集成式分词法</p>
-<p>该方法首先启动神经网络进行分词，当神经网络对新出现的词不能给出准确切分时，激活专家系统进行分析判断，依据知识库进行推理，得出初步分析，并启动学习机制对神经网络进行训练。该方法可以较充分发挥神经网络与专家系统二者优势，进一步提高分词效率。</p></li>
+<li>原来方法的计算成本比较大，在大模型上进行了优化</li>
+<li>We note that our influence analyses focus on pretrained LLMs, so our
+experiments should be interpreted as analyzing which training sequences
+contribute to a response being part of the model’s initial repertoire
+for <strong>the fine-tuning stage</strong> rather than why the final
+conversational assistant gave one response rather than another.</li>
 </ul>
-<h2 id="损失函数">损失函数</h2>
-<p><a
-href="https://zhuanlan.zhihu.com/p/35027284?utm_source=com.tencent.tim&amp;utm_medium=social">损失函数整理</a></p>
-<h2 id="补充">补充</h2>
-<p><span id="hanming"></p>
-<h3 id="汉明距离">汉明距离</h3>
-<p>在<a
-href="https://zh.wikipedia.org/wiki/%E4%BF%A1%E6%81%AF%E8%AE%BA">信息论</a>中，两个等长<a
-href="https://zh.wikipedia.org/wiki/%E5%AD%97%E7%AC%A6%E4%B8%B2">字符串</a>之间的<strong>汉明距离</strong>（Hamming
-distance）是两个字符串对应位置的不同字符的个数。换句话说，它就是将一个字符串变换成另外一个字符串所需要替换的字符个数。</p>
-<p><strong>汉明重量</strong>是字符串相对于同样长度的零字符串的汉明距离，也就是说，它是字符串中非零的元素个数：对于<a
-href="https://zh.wikipedia.org/wiki/%E4%BA%8C%E8%BF%9B%E5%88%B6">二进制</a><a
-href="https://zh.wikipedia.org/wiki/%E5%AD%97%E7%AC%A6%E4%B8%B2">字符串</a>来说，就是1的个数，所以11101的汉明重量是4。</p>
-<p></span></p>
-<p><span id="hash"></p>
-<h3 id="hash函数">hash函数</h3>
-<p>hash（散列、杂凑）函数，是将任意长度的数据映射到有限长度的域上。直观解释起来，就是对一串数据m进行杂糅，输出另一段固定长度的数据h，作为这段数据的特征（指纹）。</p>
-<p></span></p>
-<p><span id="frequency"></p>
-<h3 id="频率与频度">频率与频度</h3>
-<p>频率：一定时间（一分钟，一秒，一小时）内，某动作的次数（某事物出现的次数）。</p>
-<p>频度：即时间频度。一个算法执行所耗费的时间。一个算法中的语句执行次数称为语句频度或时间频度。</p>
-<p></span></p>
-<p><span id="Interacting"></p>
-<h3 id="互现信息">互现信息</h3>
-<p>互现信息体现了汉字之间结合关系的紧密程度。当紧密程度高于某一个阈值时，便可以认为此字组可能构成了一个词。</p>
-<p></span></p>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-  </entry>
-  <entry>
-    <title>记一次微信公众号爬虫的经历（微信文章阅读点赞的获取）</title>
-    <url>/hexo_blog/2017/11/18/%E8%AE%B0%E4%B8%80%E6%AC%A1%E5%BE%AE%E4%BF%A1%E5%85%AC%E4%BC%97%E5%8F%B7%E7%88%AC%E8%99%AB%E7%9A%84%E7%BB%8F%E5%8E%86%EF%BC%88%E5%BE%AE%E4%BF%A1%E6%96%87%E7%AB%A0%E9%98%85%E8%AF%BB%E7%82%B9%E8%B5%9E%E7%9A%84%E8%8E%B7%E5%8F%96%EF%BC%89/</url>
-    <content><![CDATA[<p>如何爬取微信公众号所有历史文章的阅读量和点赞量？该文介绍了两种突破难点的方法：一是使用微信个人订阅号进行爬取，二是在电脑上使用Fiddler进行抓包分析。详细的操作步骤及代码实现都在文中。缺陷是需要手动获取一些关键参数。针对被封禁的问题，文中提供了两种解决方案。</p>
-<span id="more"></span>
-<p>项目实现地址。查看我的<a
-href="https://github.com/wnma3mz/wechat_articles_spider">Github地址</a></p>
-<p><a href="https://data.wnma3mz.cn/">数据展示（完整指标）</a></p>
-<p><a
-href="https://data.wnma3mz.cn/demo.html">日更两次，获取公众号的最新文章链接</a>，暂不支持实时获取阅读点赞</p>
-<p>支持下载文章内容为HTML（含图片，经过二次处理可转换为PDF、Word）。欢迎技术交流，微信：wnma3mz，烦请进行相关备注，如hexo_wechat_spider。二维码见文末，样例图：</p>
-<p>目前具体可获取指标（包含但不局限）：阅读数、点赞数（在看）、评论内容及总数、正文内容及图片、是否为头条、是否为原创。
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/20200319174821639.png" /></p>
-<p>项目已经实现，比本文介绍的更加方便简单，可直接调用。</p>
-<p>关于批量关注公众号的问题已解决，见我的另一篇文章:<a
-href="https://blog.csdn.net/wnma3mz/article/details/105448808">自动批量关注微信公众号（非逆向）</a></p>
-<p>不求完美，只求能够用。。。</p>
-<p>截至2019年4月项目可正常运行， 方法已更新。</p>
-<p>3月1号更新：</p>
-<ol type="1">
-<li>获取阅读点赞时，每篇文章间隔10s 一次性可获取500篇文章以上</li>
-<li>从公众号获取永久链接时，间隔3分钟，可以连续获取几小时（网友测试）</li>
-</ol>
-<p>公开已爬取的公众号历史文章的永久链接，数据上传至GitHub，日期均截止commit时间。</p>
-<p><a
-href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/%E7%A7%91%E6%8A%80%E7%BE%8E%E5%AD%A6.json">科技美学</a></p>
-<p><a
-href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/%E5%85%B1%E9%9D%92%E5%9B%A2%E4%B8%AD%E5%A4%AE.json">共青团中央</a></p>
-<p><a
-href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/AppSo.json">AppSo</a></p>
-<p><a
-href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/%E5%8D%97%E6%96%B9%E5%91%A8%E6%9C%AB.json">南方周末</a></p>
-<h2 id="需求">需求</h2>
-<p>某某微信公众号历史的所有文章的阅读数和点赞数</p>
-<h2 id="难点">难点</h2>
-<ol type="1">
-<li>微信公众号历史的所有文章（来源？？？）</li>
-<li>每篇文章的阅读量和点赞量（电脑上浏览文章只显示内容，没有阅读量、点赞量、评论……）</li>
-</ol>
-<h3 id="突破难点一">突破难点一</h3>
-<ol type="1">
-<li><a
-href="http://weixin.sogou.com/">搜狗微信搜索</a>，可以搜索微信公众号文章。但是貌似只能显示该公众号最近十篇的文章。放弃……</li>
-<li>利用抓包工具（Fiddler），抓取文章。成本有点大……，且貌似只能抓取原创文章。不符合个人需求，放弃……</li>
-<li>利用微信个人订阅号进行爬取，神奇的操作。</li>
-</ol>
-<h4 id="操作">操作</h4>
-<ol type="1">
-<li>拥有一个微信个人订阅号，附上登陆和注册链接。<a
-href="https://mp.weixin.qq.com/">微信公众平台</a></li>
-<li>好在之前无聊注册过一次，所以就可以直接登陆操作。没有注册的童鞋可以用自己的微信号注册一下，过程十分简单，在此就不赘述了</li>
-<li>登陆之后，点击左侧菜单栏“管理”-“素材管理”。再点击右边的“新建图文素材”</li>
-</ol>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxMzExMzM2.png" /></p>
-<ol start="4" type="1">
-<li>弹出一个新的标签页，在上面的工具栏找到“超链接”并点击 <img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxNDM5MzM1.png" /></li>
-<li>弹出了一个小窗口，选择“查找文章”，输入需要查找的公众号，这里用“科技美学”公众号作为例子
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxNTM4MTk5.png" /></li>
-<li>点击之后，可以弹出该公众号的所有历史文章</li>
-</ol>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxNzA1NzQy.png" /></p>
-<h4 id="代码实现">代码实现</h4>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 目标url</span></span><br><span class="line">url = <span class="string">&quot;https://mp.weixin.qq.com/cgi-bin/appmsg&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用Cookie，跳过登陆操作</span></span><br><span class="line">headers = &#123;</span><br><span class="line">  <span class="string">&quot;Cookie&quot;</span>: yourcookie,</span><br><span class="line">  <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36&quot;</span>,</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">需要提交的data</span></span><br><span class="line"><span class="string">以下个别字段是否一定需要还未验证。</span></span><br><span class="line"><span class="string">注意修改yourtoken,number</span></span><br><span class="line"><span class="string">number表示从第number页开始爬取，为5的倍数，从0开始。如0、5、10……</span></span><br><span class="line"><span class="string">token可以使用Chrome自带的工具进行获取</span></span><br><span class="line"><span class="string">fakeid是公众号独一无二的一个id，等同于后面的__biz</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line">data = &#123;</span><br><span class="line">    <span class="string">&quot;token&quot;</span>: yourtoken,</span><br><span class="line">    <span class="string">&quot;lang&quot;</span>: <span class="string">&quot;zh_CN&quot;</span>,</span><br><span class="line">    <span class="string">&quot;f&quot;</span>: <span class="string">&quot;json&quot;</span>,</span><br><span class="line">    <span class="string">&quot;ajax&quot;</span>: <span class="string">&quot;1&quot;</span>,</span><br><span class="line">    <span class="string">&quot;action&quot;</span>: <span class="string">&quot;list_ex&quot;</span>,</span><br><span class="line">    <span class="string">&quot;begin&quot;</span>: number,</span><br><span class="line">    <span class="string">&quot;count&quot;</span>: <span class="string">&quot;5&quot;</span>,</span><br><span class="line">    <span class="string">&quot;query&quot;</span>: <span class="string">&quot;&quot;</span>,</span><br><span class="line">    <span class="string">&quot;fakeid&quot;</span>: yourfakeid,</span><br><span class="line">    <span class="string">&quot;type&quot;</span>: <span class="string">&quot;9&quot;</span>,</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用get方法进行提交</span></span><br><span class="line">content_json = requests.get(url, headers=headers, params=data).json()</span><br><span class="line"><span class="comment"># 返回了一个json，里面是每一页的数据</span></span><br><span class="line"><span class="keyword">for</span> item <span class="keyword">in</span> content_json[<span class="string">&quot;app_msg_list&quot;</span>]:</span><br><span class="line">    <span class="comment"># 提取每页文章的标题及对应的url</span></span><br><span class="line">    <span class="built_in">print</span>(item[<span class="string">&quot;title&quot;</span>], <span class="string">&quot;url&quot;</span>: item[<span class="string">&quot;link&quot;</span>])</span><br></pre></td></tr></table></figure>
-<p>以上，即可爬取微信公众号的一页数据，如果是爬取所有页的数据，则需要改变number进行爬取。</p>
-<p>注：每次抓取完一页之后，最好设定time.sleep(3)。过快会导致爬取失败</p>
-<h3 id="突破难点二">突破难点二</h3>
-<p>这里我使用的方法是在电脑上登陆微信客户端，进行抓包分析。从客户端看推文可以看到阅读量、点赞量。</p>
-<p>我使用的是Fiddler。Fiddller具体使用就不赘述了。下面直接演示操作</p>
-<h4 id="操作-1">操作</h4>
-<ol type="1">
-<li>打开fiddler开始监控</li>
-<li>登陆微信客户端，浏览该公众号的任意一篇推文</li>
-<li>可以观察到这里的内容显示会有阅读量、点赞量、评论等</li>
-<li>观察fiddler的监控数据，如下图显示</li>
-</ol>
+<h2 id="method-simple">Method (Simple)</h2>
+<p>假设有这么一个函数能够计算每条训练数据对于当前生成结果的影响分数</p>
+<p>输入：Prompt + Completion，所有的训练数据、模型</p>
+<p>输出：每条训练数据的分数</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxODMzNzAx.png" />
-5. 其中
-<code>/mp/getappmgsext?...</code>是我们推文内容的url，双击之后，fiddler界面右边出现如下图数据</p>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/625460f4950846b89c4b247deb2c5cd2.webp" /></p>
+<p>Prompt + Completion 可以在训练数据中，也可以不在</p>
+<h2 id="resultmodel-scale">Result——Model Scale</h2>
+<p>对于简单的事实查询，前100个有影响的序列通常包含正确完成所有模型之间关系所需的信息。</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
+<tr>
+<th>0.81B Model (Influence = 0.122)</th>
+<th>52B Model (Influence = 0.055)</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/b128475ec4bb4596a8758f1c5ff35b3d.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/71a4844c619c4d29b6ca893487f8ac91.webp" /></td>
+</tr>
+</tbody>
+</table>
+<p>红色和绿色分别表示对句子产生正面和负面影响的 token</p>
+<h3 id="q-a">Q &amp; A</h3>
+<p>小模型仅仅是根据字词相关来作出响应，而大模型是根据主题/语义来作出响应</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
+<tr>
+<th>0.81B Model (Influence = 0.681)</th>
+<th>52B Model (Influence = 0.126)</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/6443c3a251734449b5ae927ca1fa09c1.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/00473a7fae4e4174bea1b799d498974d.webp" /></td>
+</tr>
+</tbody>
+</table>
+<h3 id="q-a-1">Q &amp; A</h3>
+<p>小模型集中在字词相似，而大模型能够 get 到讽刺这种语境</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
+<tr>
+<th>0.81B Model</th>
+<th>52B Model</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2f87cff844e74a838deb16d3501e20ec.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/a82fa4e8b3ea48f4bb1de6d2375a9c92.webp" /></td>
+</tr>
+</tbody>
+</table>
+<h3 id="math">Math</h3>
+<p>小模型专注于 clips 单词，而大模型是相关问题的。（混淆了变量名）</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
+<tr>
+<th>0.81B Model</th>
+<th>52B Model</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e74e8d2f49274a478703c632043e31db.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/f90974c1c90c4a3082e8572f6dbcf0d7.webp" /></td>
+</tr>
+</tbody>
+</table>
+<h3 id="code">Code</h3>
+<p>混淆了 Prompt 和 Completion 的变量名</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
+<tr>
+<th>0.81B Model</th>
+<th>52B Model</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/c0eab626552c4eda9df4ee4ce908089c.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e54956f34cbc406d946e092d4a803caa.webp" /></td>
+</tr>
+</tbody>
+</table>
+<h3 id="cross-language">Cross-Language</h3>
+<p>把 Prompt 和 Completion
+换成其他语言（韩语和土耳其语）。选出影响分数的十条数据作为第一行的
+Sequences。总共三行，后两行也是这十条数据。</p>
+<p>随着模型规模增大，跨语言“检索”能力逐渐增强。</p>
+<h2 id="resultlocalizing-influence">Result——<strong>Localizing
+Influence</strong></h2>
+<p><strong>For 52B Model</strong></p>
+<p>影响函数还可以计算每一层的影响分数</p>
+<h3 id="layerwise-influence-distribution">Layerwise influence
+distribution</h3>
+<p>横坐标：每个主题里面最有影响力的 500 条数据</p>
+<p>纵坐标：网络的浅层和深层</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxOTA0MDAw.png" />
-6. 上图下侧的json里面的
-<code>read_num</code>、<code>like_num</code>分别是阅读量和点赞量</p>
-<p>####代码实现</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 目标url</span></span><br><span class="line">url = <span class="string">&quot;http://mp.weixin.qq.com/mp/getappmsgext&quot;</span></span><br><span class="line"><span class="comment"># 添加Cookie避免登陆操作，这里的&quot;User-Agent&quot;最好为手机浏览器的标识</span></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;Cookie&quot;</span>: yourcookie,</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat QBCore/3.43.27.400 QQBrowser/9.0.2524.400&quot;</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">data = &#123;</span><br><span class="line">    <span class="string">&quot;is_only_read&quot;</span>: <span class="string">&quot;1&quot;</span>,</span><br><span class="line">    <span class="string">&quot;is_temp_url&quot;</span>: <span class="string">&quot;0&quot;</span>,              </span><br><span class="line">    <span class="string">&quot;appmsg_type&quot;</span>: <span class="string">&quot;9&quot;</span>, <span class="comment"># 新参数，不加入无法获取like_num</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">添加请求参数</span></span><br><span class="line"><span class="string">__biz对应公众号的信息，唯一</span></span><br><span class="line"><span class="string">mid、sn、idx分别对应每篇文章的url的信息，需要从url中进行提取</span></span><br><span class="line"><span class="string">key、appmsg_token从fiddler上复制即可</span></span><br><span class="line"><span class="string">pass_ticket对应的文章的信息，貌似影响不大，也可以直接从fiddler复制</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line">params = &#123;</span><br><span class="line">    <span class="string">&quot;__biz&quot;</span>: your__biz,</span><br><span class="line">    <span class="string">&quot;mid&quot;</span>: article_mid,</span><br><span class="line">    <span class="string">&quot;sn&quot;</span>: article_sn,</span><br><span class="line">    <span class="string">&quot;idx&quot;</span>: article_idx,</span><br><span class="line">    <span class="string">&quot;key&quot;</span>: yourkey,</span><br><span class="line">    <span class="string">&quot;pass_ticket&quot;</span>: pass_ticket,</span><br><span class="line">    <span class="string">&quot;appmsg_token&quot;</span>: yourappmsg_token,</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment"># 使用post方法进行提交</span></span><br><span class="line">content = requests.post(url, headers=headers, data=data, params=params).json()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 由于上面这种方法可能会获取数据失败，可以采取字符串拼接这种方法</span></span><br><span class="line">origin_url = <span class="string">&quot;https://mp.weixin.qq.com/mp/getappmsgext?&quot;</span></span><br><span class="line">appmsgext_url = origin_url + <span class="string">&quot;__biz=&#123;&#125;&amp;mid=&#123;&#125;&amp;sn=&#123;&#125;&amp;idx=&#123;&#125;&amp;appmsg_token=&#123;&#125;&amp;x5=1&quot;</span>.<span class="built_in">format</span>(your__biz, article_mid, article_sn, article_idx, yourappmsg_token)</span><br><span class="line">content = requests.post(appmsgext_url, headers=headers, data=data).json()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取其中的阅读数和点赞数</span></span><br><span class="line"><span class="built_in">print</span>(content[<span class="string">&quot;appmsgstat&quot;</span>][<span class="string">&quot;read_num&quot;</span>], content[<span class="string">&quot;appmsgstat&quot;</span>][<span class="string">&quot;like_num&quot;</span>])</span><br></pre></td></tr></table></figure>
-<p>以上即可获取到一篇文章的阅读量、点赞量。
-<code>Cookie</code>、<code>req_id</code>、<code>pass_ticket</code>、<code>key</code>、<code>appmsg_token</code>、<code>__biz</code>利用fiddler获取
-如果是需要多篇文章，需要更改的参数
-<code>mid</code>、<code>sn</code>、<code>idx</code>
-如果是不同公众号，就需要根据url修改 <code>__biz</code>
-多篇文章的爬取，需要根据之前爬取到的url进行提取关键信息，分别爬取。
-注：每次抓取完一页之后，最好设定time.sleep(3)。过快会导致爬取失败。</p>
-<h2 id="写在最后">写在最后</h2>
-<p>以上就是这次微信爬虫的经历。</p>
-<p>需要掌握的基本技能：</p>
-<ol type="1">
-<li>python爬虫的语法</li>
-<li>Chrome、Fiddler基本使用</li>
-<li>网络爬虫基本常识</li>
-</ol>
-<p>缺陷：</p>
-<ol type="1">
-<li>使用Cookie登陆，未实现自动登陆</li>
-<li>key、appmsg_token一些关键参数需要进行手动获取</li>
-<li>实际运行之后，就算设定了爬取间隙时间，还是会被封禁（获取链接时）。</li>
-</ol>
-<p>说明：</p>
-<ol type="1">
-<li>网上一些说法，key半小时过期，我好像没有遇到。</li>
-<li>代码中若有细节问题，欢迎指出讨论</li>
-</ol>
-<p>Github上已经实现了第1、2点，欢迎回到文章开头看github上的实现过程。</p>
-<p>关于被封禁的问题，已有两个解决方案，均放在github上，在这不做讲解。</p>
-<ol type="1">
-<li>selenium解决方案，提高约十倍抓取量（不确定）。优点：提高抓取量；缺点：速度慢，不一定能完全抓取完整，抓取量不确定。<a
-href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/test/test_seleinum/test_seleinum.py">test_seleinum.py</a></li>
-<li>利用个人微信号的查看公众号历史消息，抓取量在500条以上，具体未测试。优点：抓取量最多的方案；缺点：短时间内（5-10分钟）无法查看历史消息，或者说无法持续抓取；不保证微信号会被封号。<a
-href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/test/test_GetUrls.py">test_GetUrls.py</a></li>
-</ol>
-]]></content>
-      <categories>
-        <category>Scrapy</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>爬虫</tag>
-        <tag>微信</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>论文从零到一</title>
-    <url>/hexo_blog/2019/05/28/%E8%AE%BA%E6%96%87%E4%BB%8E%E9%9B%B6%E5%88%B0%E4%B8%80/</url>
-    <content><![CDATA[<p>如何写出一篇论文</p>
-<span id="more"></span>
-<h2 id="论文从零到一">论文从零到一</h2>
-<p>所谓工欲善其事必先利其器，开头先介绍一些工具与技能（皆非必需）。</p>
-<h3 id="工具与技能">工具与技能</h3>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2c0084f89e9547409d6aa78ccc80fa68.webp" /></p>
+<p>小结：</p>
 <ul>
-<li>LaTex（论文写作工具，当然Word也行），以下是选用工具
+<li>补全句子（简单任务），upper layers</li>
+<li>数学和编程：middle layers</li>
+<li>翻译：middle layers</li>
+<li>记忆：upper layers</li>
+<li>角色扮演：middle layers (with some influences concentrated in the
+lower and upper layer)</li>
+</ul>
+<p>The 810 million parameter model exhibited roughly similar patterns,
+but with <strong>less consistency</strong>.</p>
+<p>分布不像 52B 模型这么集中在某些层。</p>
+<p>猜测：</p>
 <ul>
-<li><a href="http://www.tablesgenerator.com/latex_tables">Create LaTeX
-tables online</a></li>
-<li><a href="https://www.overleaf.com/?&amp;nocdn=true">Overleaf, Online
-LaTeX Editor</a></li>
-<li><a href="http://www.latextemplates.com/">LaTeX Templates</a></li>
+<li>lower layers是接近输入层的部分，upper layers是接近输出层的部分</li>
+<li>为什么角色扮演会有差异，话题可能可能包含数学、记忆等其他类型的任务</li>
+</ul>
+<h3 id="limit-different-layers">Limit Different layers</h3>
+<p>固定某些层，再计算最具影响力的训练数据</p>
+<p>To further investigate the localization of influence to different
+layers, we computed the most influential sequences when the influence
+was restricted to the lower, middle, or upper layers.</p>
+<p>Influential sequences only computed on <strong>the middle layers were
+generally more thematically</strong> related to the query (also with
+less sparse tokenwise distribution). <a
+href="https://arxiv.org/abs/2202.05262">https://arxiv.org/abs/2202.05262</a></p>
+<table>
+<colgroup>
+<col style="width: 7%" />
+<col style="width: 30%" />
+<col style="width: 30%" />
+<col style="width: 30%" />
+</colgroup>
+<thead>
+<tr>
+<th></th>
+<th>Upper layers</th>
+<th>Middle layers</th>
+<th>Lower layers</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>superintelligent（角色扮演）</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5d093adc546d4bf89b4c0c28a6c5f078.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/32cead3f3d8a4b93bba4cc9cf286835c.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8b0a3c16b1a844a291bf2507653f7b5d.webp" /></td>
+</tr>
+<tr>
+<td>inflation （简单补全）</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/ceb342ca571645129c0717897d605406.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/c44c8253e03a45bf803db7b919b84836.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/88a731f7a8004cc7aea38d11497c0d57.webp" /></td>
+</tr>
+</tbody>
+</table>
+<h2 id="resultmemorization">Result——Memorization</h2>
+<p><strong>LLM 是不是直接记忆并复述特定的训练序列</strong></p>
+<h3 id="例子">例子</h3>
+<p>训练数据有下面三个句子 A: A1+A2+A3</p>
+<p>B: B1+B2+B3</p>
+<p>C: C1+C2+C3</p>
+<p>当输入是 A1 时，输出是 A2 吗？</p>
+<ul>
+<li><p>实验（口头描述）：</p>
+<ul>
+<li>We have examined numerous examples of the AI Assistant’s outputs and
+(with the exception of famous quotes or passages targeting memorization,
+as described below) have <strong>not</strong> been able to identify
+clear instances of <strong>memorization</strong>, such as copying an
+entire sentence or copying the flow of ideas in an entire paragraph. We
+also did not observe cases where a single sequence dominated the
+influence.</li>
 </ul></li>
-<li>Visio（微软家的画图工具，主要是流程图之类的，当然PPT也行）</li>
-<li>PhotoShop（Emmm，我反正不用）</li>
-<li>Matlab或Python的Matplotlib（绘制展现数据的图表，当然Origin也行）</li>
-<li>Zotero（文献整理工具，当然不用也行）</li>
-<li>SumatraPDF或Adobe Acrobat DC（PDF阅读器）</li>
-<li>知网（论文启蒙网站。。。更多的推荐谷歌学术，Web Of
-Science与Arxiv）</li>
-<li>Sci-Hub或Sci-Hub的telegram bot，同时推荐<a
-href="https://lunwen.im/">https://lunwen.im/</a>（论文下载）</li>
-<li>天若OCR文字识别（OCR文字识别）</li>
-<li>CopyTranslator（复制翻译）</li>
-<li>Mathpix
-Snip（图片转公式，这个需要订阅了，手机端还是免费的，绕过方法见附录）</li>
-<li>1Checker和Grammarly（语法检查工具）</li>
 </ul>
-<p>以上除PhotoShop外，其余我在用，当然大部分其实不用或用其他的也行。说白了单凭Word能不能写出好论文，那肯定是可以的。这些只不过是外力罢了，主要还是看自己。</p>
-<h3 id="正文">正文</h3>
-<h4 id="文献调研">文献调研</h4>
-<p>诸如联系老师和确定方向之类的在这里就不谈了，从文献检索开始介绍。如果是一个未接触过的方向，如何从零开始学习呢。这里如果是导师给出一个论文大礼包自然是最好，但我自己的经历告诉我还是自己动手丰衣足食。我自己的方法是，从知网开始学习，当然如果英语足够的话，可以直接跳过此步骤。知网检索相关关键词，筛选出硕士论文与博士论文，下载20篇（一页左右）。这里题外话，关于知网PDF格式下载问题，请使用<a
-href="http://gb.oversea.cnki.net/kns55/default.aspx">海外知网</a>，如果学校没版权的话请另寻他路（方法很多在此不介绍，个人还是应该具备基本的检索能力）。</p>
-<p>为什么选择学位论文呢？因为中国的学位论文是用中文写的，看得懂且质量上有一定的保障。学位论文往往是一个研究生几年内几个工作的总结，故在背景与方法介绍上会更为全面，语言对于普通读者更为友好，专业术语大多都进行了解释。所以在阅读时，首先可以直接从头开始阅读（甚至可以略去摘要），了解清楚一个行业背景和相关方法后，后面阐述作者的工作（文章主体）可以暂时略去不看。在阅读完20篇（甚至不需要20篇）后，基本可以了解清楚该方向的背景与相关知识，到此为止相当于进行了一个方向的科普工作。更进一步，需要对20篇中感兴趣的工作进行更进一步的阅读与了解，此时就需要进行文献整理，提取有效信息。这样可以学习到别人的方法与应用场景，甚至可以尝试去复现，提出自己的观点。</p>
-<p>第一步基本到此结束，但这还远远不够。哪怕是最新的中文文章，可能离国际前沿还是具有一定的差距，且有些方向很新的工作，国内相关论文还比较少。所以，接下来就需要学习英文论文。使用<a
-href="https://scholar.google.cn/">谷歌学术</a>或<a
-href="http://apps.webofknowledge.com/UA_GeneralSearch_input.do?product=UA&amp;search_mode=GeneralSearch&amp;SID=8BsQBbbIKZlBjqwLoEq&amp;preferencesSaved=">Web
-Of
-Science</a>，同样也是搜索关键词（英文），或者根据前面阅读的学位论文查看哪些英文文献经常被引用，可以直接搜索相关文献。除了直接搜索关键词，也可以考虑以下几个点来进行搜索（虽然我自己不怎么用这个方法），已知某个教授在该方向有所建树，故可以直接搜索该教授名字；已知某高校学院在此方向处于国际领先水平，可以直接搜索机构等。在得到相关几篇英文论文后，可以根据引用的论文来回溯更早之前的论文研究，也可以借助如谷歌学术等查看某篇文章被哪些文章引用，可以知道该篇文章的方法最新进展（有可能在某个时间段就被研究人员放弃了）。这样在阅读几十篇英文文章后，就能了解到该方向的最新进展与成果（或者直接筛选时间节点为最近一年或半年的已发表论文）。当然，在一开始阅读英文文献时确实会比较吃力，但是在前期有中文文献阅读的基础上（文章会夹杂着专业术语），这样阅读起来还是比较容易的，把领域内名词吃透后，接下来不过是一些动词形容词而已。学习曲线大概是先难后易的一个过程，一开始需要对某篇（或某几篇）优秀论文进行精读，花费大量时间去翻译研究复现，后续就可以泛读了。关于文章阅读量的问题，贵精不贵多，不一定真的要阅读到很多篇论文才能开始自己的工作。</p>
-<p>下一阶段开始前，建议对阅读的文献进行整理，推荐使用<strong>Zotero</strong>，配合坚果云更香。具体使用方法自行检索。还有Arxiv的话，也可以作为文献检索工具来使用，但是有限定专业人群。</p>
-<h4 id="实验">实验</h4>
-<p>其实关于实验部分可介绍的东西，每个专业甚至每个方向的实验内容都不大一样。所以在这里给出的建议就是有问题及时与老师反馈，跟相关方向的师兄师姐交流。如果是硬件资源问题的话，那我也。。。嗯，关于深度学习的话，可以暂时勉强用Google的Colab。还有诸如网络环境这种，各种工具使用这种，还是需要慢慢踩坑的。这里我有几个人的小Trick可以分享出来。</p>
+<h3 id="影响力分布的定量实验">影响力分布的定量实验</h3>
+<p>动机：影响有多集中也就是说，每个模型的输出是否主要来自少量的训练序列还是它结合了来自许多不同序列的信息？</p>
+<p>方法：</p>
 <ul>
-<li>谷歌云盘文件获取直链
+<li>给每个句子算好分之后，计算每个句子的概率值</li>
+<li>用几种分布分别去拟合实际的概率分布，发现<strong>幂律分布</strong>最符合</li>
+<li>这种参数分布形式通常用于建模长尾行为（二八定律）</li>
+</ul>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/2642b83f1b414e1584b586072c945160.webp" /></p>
 <ul>
-<li>共享文件，获取文件id</li>
-<li>海外服务器，获取转换脚本<code>wget --no-check-certificate -qO /usr/local/bin/gdlink 'https://moeclub.org/attachment/LinuxShell/gdlink.sh' &amp;&amp; chmod a+x /usr/local/bin/gdlink</code></li>
-<li>运行gdlink，获取直链接。gdlink
-'18v_9Bet11B3ZZ3GqWCOa3yTuY2Lzzaav'。输出可直接下载</li>
+<li>会不会是影响函数无法检测到？</li>
+</ul>
+<h3 id="检验实验">检验实验</h3>
+<ul>
+<li>用训练数据的原始句子检验</li>
+<li>验证了当存在明确记忆的情况时，影响分数最大的句子是原文→影响函数能够匹配</li>
+</ul>
+<p>For 52B Model</p>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/dda0d64781cd4552b8e18e722d11336e.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/06ab6f2d1d894063a1fd832723771267.webp" /></td>
+</tr>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/fdd9eb4647b542a0a93cdffa2280243b.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/f9c95ee4e69e47d082654db9d2e701a1.webp" /></td>
+</tr>
+</tbody>
+</table>
+<ul>
+<li>结论：
+<ul>
+<li>不太可能是直接记忆训练序列，而是源自许多训练序列的集合</li>
+<li>无法排除模型以更隐晦的方式记忆了训练数据</li>
 </ul></li>
-<li>海外服务器可以利用Github的学生包薅一下<strong>digitalocean</strong>的羊毛，时长一年左右；微软的貌似也可以，还没试。</li>
-<li>关于内网的问题，如果是局域网还好说，可以使用VSCode的插件编辑代码，传输文件也比较方便。如果不是局域网的话，可以参考我的<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>
-使用ssh进行配置，几行命令就ok了，我感觉比一般的内网穿透方便很多。当然需要一个中转服务器这种。</li>
 </ul>
-<h4 id="写作与投稿">写作与投稿</h4>
-<p>写作部分，大部分都可以从网上搜得到。这里给个建议是说，可以先从模仿开始，不是复制粘贴，是模仿其他论文的格式以及图表形式（用的什么图说明的什么问题）。写作工具还是推荐使用LaTex，最省心的其实就两点排版与插入参考文献。当然还有很多其他优点，比如数学公式的书写。这里引用文献的话，就体现出了为什么要整理文献这一步骤，如果借助文献整理工具，更是可以减少很多不必要的时间。如果没有工具的话，就直接使用谷歌学术导出引用格式（具体自行检索）；有工具的话，可以直接利用工具一键导出，无论是word写作还是LaTex写作。</p>
-<p>图片一律推荐使用svg格式，但是由于LaTex不直接支持svg，故需要换成eps格式。绘图工具在前面也进行了介绍，自行学习。通篇文章写完之后，建议是找使用英文为母语的人进行阅读（这一点挺难的对于大多国内学生），所以可以使用语法检查工具（在前面也有介绍）。还有一些词汇的使用，避免某些词高频使用，查阅同义词词典进行替换，也可以提高论文的语言水平。</p>
-<p>投稿的话，第一步就是期刊选择的问题。这个一般可以由老师推荐，如果没有推荐的话，可以使用<a
-href="http://www.letpub.com.cn/index.php?page=journalapp&amp;view=search">lethub</a>进行查找相关方向的期刊并且利用<a
-href="https://www.scimagojr.com/journalrank.php">journalrank</a>全面的查找IF和分区等。这里值得注意的是投稿时间与投稿难度仅做部分参考，可信度有待商榷，更科学地方法是到期刊官网看最近的收录期刊以及收录论文的方向。关于OA的问题，OA，OpenAccess，开放期刊，读者可免费阅读期刊内的论文。所以此类期刊的收录一般更大，审稿速度更快，但与之相对的版面费更高，根据自己的实际期刊慎投。一般来说，期刊提交初稿都比较随意，可以直接提交pdf或docx，可以不提交原始稿件这种。后续要修改的话，比如LaTex官网会有对应的模板文件可以直接引用，十分方便。Word的话不是很清楚，可能编辑会帮忙。关于推荐审稿人的问题，有些期刊在提交时会要求推荐审稿人，这里应该是可以随意的，也不一定真的是推荐审稿人审稿，故这里可以推荐关系好的朋友。。。关于CoverLetter的问题，有的期刊是需要提交CoverLetter，类似于自荐信这种，格式网上也基本找得到。基本内容大概就是论文内容以及个人联系方式，为什么要投该期刊等。</p>
-<p>审稿后，返回的意见也就那几种，具体意思可以查阅网上相关内容。这里仅提一点就是，审稿意见需要客观看待，不是拒了就表示文章不行，需要坚持投稿，根据审稿意见改进后，投其他期刊。关于一稿多投的问题，这种问题还是不要出现，不然会对论文的作者们产生不良影响。</p>
-<p>祝各位论文高效产出，投稿必中！</p>
-<p>P.S. 如还有相关疑惑，欢迎留言or邮件wnma3mz@gmail.com！</p>
-<h3 id="附录">附录</h3>
-<h4 id="latex">LaTex</h4>
-<p><strong>LaTeX</strong>，是一种基于<a
-href="https://zh.wikipedia.org/wiki/TeX">TeX</a>的<a
-href="https://zh.wikipedia.org/wiki/排版">排版</a>系统，由<a
-href="https://zh.wikipedia.org/wiki/美国">美国</a><a
-href="https://zh.wikipedia.org/wiki/计算机科学">计算机科学</a>家<a
-href="https://zh.wikipedia.org/wiki/莱斯利·兰伯特">莱斯利·兰伯特</a>在20世纪80年代初期开发，利用这种格式系统的处理，即使使用者没有排版和程序设计的知识也可以充分发挥由TeX所提供的强大功能，不必一一亲自去设计或校对，能在几天，甚至几小时内生成很多具有书籍品质的印刷品。对于生成复杂表格和<a
-href="https://zh.wikipedia.org/wiki/数学">数学</a>公式，这一点表现得尤为突出。因此它非常适用于生成高印刷质量的<a
-href="https://zh.wikipedia.org/wiki/科技">科技</a>和数学、物理文档。这个系统同样适用于生成从简单的信件到完整书籍的所有其他种类的文档。（摘自Wikipedia）</p>
-<p>简单总结，LaTex是一种很nb的排版软件。对于数学、物理、计算机等需要大量数学公式的相关专业学生，强烈建议学习。入门书籍请参考：<a
-href="https://github.com/wnma3mz/Reading-Books/blob/master/非技术/latex教程中文.pdf">latex教程中文.pdf</a>与<a
-href="https://github.com/wnma3mz/Reading-Books/blob/master/非技术/一份不太简短的%20LATEX%202ε%20介绍.pdf">一份不太简短的
-LATEX 2ε 介绍.pdf</a></p>
-<p>几大不得不说的好处：</p>
-<ol type="1">
-<li>引用参考文献，只需要一个<code>bib</code>文件，参考文献风格只需要一行命令便可定义</li>
-<li>数学公式的书写</li>
-<li>各种排版可随时切换</li>
-</ol>
-<h4 id="绘图相关">绘图相关</h4>
-<p>在一篇论文，好看的图表和不可缺少的。大部分数据图用 Matplotlib
-即可完成，流程图也用Visio进行绘制（甚至使用PPT），最后可以考虑使用PS进行完善</p>
-<h4 id="zotero">Zotero</h4>
-<p>开源的文献整理工具。对我而言，它的优势在于：</p>
-<ol type="1">
-<li>云同步（配合坚果云，无限量同步）</li>
-<li>自由上传文件并自动查找文献索引</li>
-<li>一键导出参考文献</li>
-<li>创建时间轴</li>
-<li>批量整理文件</li>
-</ol>
-<h4 id="mathpix-snip">Mathpix Snip</h4>
-<p>对于一个长且复杂的数学公式，我是绝对不想自己重新手写一遍的。此工具可以很方便的将图片转为LaTex公式。在2018年，此软件还是免费使用的状态。但目前已需要付费使用，免费次数于我而言远远不够。同类软件我还没找到很好的替代品。故要继续使用有两种方法，直接付费按年订阅或购买API订阅，再二次封装。均需一定的费用。无意发现安卓版本（IOS不清楚）的调用是免费的，故进行抓包获取了请求API的key，再进行二次封装。目前可稳定运行</p>
-]]></content>
-      <categories>
-        <category>Life</category>
-      </categories>
-  </entry>
-  <entry>
-    <title>温度可视化</title>
-    <url>/hexo_blog/2023/12/03/%E6%B8%A9%E5%BA%A6%E5%8F%AF%E8%A7%86%E5%8C%96/</url>
-    <content><![CDATA[<p>时序数据，使用简单的 HTML 对其进行展示。</p>
-<span id="more"></span>
-<h2 id="需求描述">需求描述</h2>
-<p>接 <a href="/hexo_blog/2023/11/26/IOT/%E6%A0%91%E8%8E%93%E6%B4%BE/%E6%A0%91%E8%8E%93%E6%B4%BE%E6%8A%98%E8%85%BE%E8%AE%B0%E5%BD%95/" title="树莓派折腾记录">树莓派折腾记录</a> 中"获取小米温湿度计"的数据,
-现在有了若干数据后，希望用一种直观的方式展示出来。</p>
-<h2 id="解决方案">解决方案</h2>
-<p>不想用复杂的前后端代码，希望用最简单的方式画一个折线图即可。所以解决方案是，用
-Python 处理日志数据，将其保存到 json 中，最后用 HTML
-结合现有的可视化库进行展示。为了有一定的实时性，所以在用 Python
-处理日志时，也需要一个定时任务，以定期更新前端页面的数据。</p>
-<p>一种不太优雅的方案，可以用 Python 的画图库，将其导出为图片，然后用
-HTML 的 img
-标签进行展示。但是这样的话，图片的大小就不好控制了，而且也不太好看。所以还是用
-HTML 的可视化库比较好。</p>
-<figure class="highlight html"><table><tr><td class="code"><pre><span class="line"><span class="meta">&lt;!DOCTYPE <span class="keyword">html</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">img</span> <span class="attr">src</span>=<span class="string">&quot;static/pi_data.png&quot;</span> <span class="attr">alt</span>=<span class="string">&quot;温度曲线&quot;</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre></td></tr></table></figure>
-<h2 id="代码实现">代码实现</h2>
-<p>用 Google 的可视化库 <a
-href="https://www.gstatic.com/charts/loader.js">https://www.gstatic.com/charts/loader.js</a>，下载后将其保存在本地。</p>
-<p>成品展示：<a
-href="https://raspberrypi.tail54f27.ts.net/temp">https://raspberrypi.tail54f27.ts.net/temp</a>。为了节约显示数据，Python
-中只处理过去 24 小时的数据</p>
-<figure class="highlight html"><table><tr><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">script</span> <span class="attr">type</span>=<span class="string">&quot;text/javascript&quot;</span> <span class="attr">src</span>=<span class="string">&quot;static/loader.js&quot;</span>&gt;</span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">script</span> <span class="attr">type</span>=<span class="string">&quot;text/javascript&quot;</span>&gt;</span><span class="language-javascript"></span></span><br><span class="line"><span class="language-javascript">    google.<span class="property">charts</span>.<span class="title function_">load</span>(<span class="string">&#x27;current&#x27;</span>, &#123; <span class="string">&#x27;packages&#x27;</span>: [<span class="string">&#x27;corechart&#x27;</span>] &#125;);</span></span><br><span class="line"><span class="language-javascript">    <span class="comment">/* </span></span></span><br><span class="line"><span class="comment"><span class="language-javascript">        读取 json 文件，约定好几个字段 data, title, left_label, right_label 即可</span></span></span><br><span class="line"><span class="comment"><span class="language-javascript">        其中，data 是一个二维数组，第一列是时间，第二列是温度，第三列是湿度。但第一行是标题 [&quot;横坐标标题&quot;, &quot;第x条线的标题&quot;]。</span></span></span><br><span class="line"><span class="comment"><span class="language-javascript">        left_label 是左边的纵坐标标题，right_label 是右边的纵坐标标题。</span></span></span><br><span class="line"><span class="comment"><span class="language-javascript">    */</span></span></span><br><span class="line"><span class="language-javascript">  </span></span><br><span class="line"><span class="language-javascript">    <span class="title function_">fetch</span>(<span class="string">&#x27;static/pi_data.json&#x27;</span>)</span></span><br><span class="line"><span class="language-javascript">      .<span class="title function_">then</span>(<span class="function"><span class="params">response</span> =&gt;</span> response.<span class="title function_">json</span>())</span></span><br><span class="line"><span class="language-javascript">      .<span class="title function_">then</span>(<span class="function"><span class="params">data</span> =&gt;</span> &#123;</span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> drawChart_callback = <span class="title function_">drawChart</span>(data.<span class="property">data</span>, <span class="string">&quot;chart&quot;</span>, data.<span class="property">title</span>, data.<span class="property">left_label</span>, data.<span class="property">right_label</span>);</span></span><br><span class="line"><span class="language-javascript">        google.<span class="property">charts</span>.<span class="title function_">setOnLoadCallback</span>(drawChart_callback);</span></span><br><span class="line"><span class="language-javascript">      &#125;)</span></span><br><span class="line"><span class="language-javascript">      .<span class="title function_">catch</span>(<span class="function"><span class="params">error</span> =&gt;</span> &#123;</span></span><br><span class="line"><span class="language-javascript">        <span class="variable language_">console</span>.<span class="title function_">error</span>(<span class="string">&#x27;读取JSON文件时出错：&#x27;</span>, error);</span></span><br><span class="line"><span class="language-javascript">      &#125;);</span></span><br><span class="line"><span class="language-javascript">    <span class="keyword">function</span> <span class="title function_">drawChart</span>(<span class="params">data_arr, chart_id, title, left_label, right_label</span>) &#123;</span></span><br><span class="line"><span class="language-javascript">      <span class="keyword">return</span> <span class="keyword">function</span> (<span class="params"></span>) &#123;</span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> data = google.<span class="property">visualization</span>.<span class="title function_">arrayToDataTable</span>(data_arr);</span></span><br><span class="line"><span class="language-javascript"></span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> options = &#123;</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">hAxis</span>: &#123;</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">useFormatFromData</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">viewWindow</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">minValue</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">maxValue</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">viewWindowMode</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">          &#125;,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">legacyScatterChartLabels</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">vAxes</span>: [&#123;</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;useFormatFromData&quot;</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;viewWindow&quot;</span>: &#123; <span class="string">&quot;max&quot;</span>: <span class="literal">null</span>, <span class="string">&quot;min&quot;</span>: <span class="literal">null</span> &#125;,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;minValue&quot;</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;maxValue&quot;</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;title&quot;</span>: left_label,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;logScale&quot;</span>: <span class="literal">false</span>,</span></span><br><span class="line"><span class="language-javascript">          &#125;, &#123;</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;useFormatFromData&quot;</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;viewWindow&quot;</span>: &#123; <span class="string">&quot;max&quot;</span>: <span class="literal">null</span>, <span class="string">&quot;min&quot;</span>: <span class="literal">null</span> &#125;,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;minValue&quot;</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;maxValue&quot;</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;title&quot;</span>: right_label,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;logScale&quot;</span>: <span class="literal">false</span>,</span></span><br><span class="line"><span class="language-javascript">          &#125;],</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">booleanRole</span>: <span class="string">&quot;certainty&quot;</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">lineWidth</span>: <span class="number">2</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">legend</span>: <span class="string">&quot;top&quot;</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">title</span>: title,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">fontName</span>: <span class="string">&quot;sans-serif&quot;</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">useFirstColumnAsDomain</span>: <span class="literal">false</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">titleTextStyle</span>: &#123; <span class="string">&quot;color&quot;</span>: <span class="string">&quot;#000&quot;</span>, <span class="string">&quot;fontSize&quot;</span>: <span class="number">24</span>, <span class="string">&quot;bold&quot;</span>: <span class="literal">true</span> &#125;,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">series</span>: &#123; <span class="string">&quot;1&quot;</span>: &#123; <span class="string">&quot;targetAxisIndex&quot;</span>: <span class="number">1</span> &#125; &#125;,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">curveType</span>: <span class="string">&quot;&quot;</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">interpolateNulls</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">series</span>: &#123;</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;2&quot;</span>: &#123;</span></span><br><span class="line"><span class="language-javascript">                <span class="string">&quot;targetAxisIndex&quot;</span>: <span class="number">1</span></span></span><br><span class="line"><span class="language-javascript">            &#125;</span></span><br><span class="line"><span class="language-javascript">          &#125; <span class="comment">/* 这个字段用于控制第几列放在右边？作为调整左右纵坐标位置的参数，2表示用第3列的数据*/</span></span></span><br><span class="line"><span class="language-javascript">        &#125;;</span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> isDefaultVisualization = <span class="literal">true</span>;</span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> chart = <span class="keyword">new</span> google.<span class="property">visualization</span>.<span class="title class_">LineChart</span>(<span class="variable language_">document</span>.<span class="title function_">getElementById</span>(chart_id));</span></span><br><span class="line"><span class="language-javascript"></span></span><br><span class="line"><span class="language-javascript">        chart.<span class="title function_">draw</span>(data, options, isDefaultVisualization);</span></span><br><span class="line"><span class="language-javascript">      &#125;</span></span><br><span class="line"><span class="language-javascript">    &#125;</span></span><br><span class="line"><span class="language-javascript">  </span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">style</span>&gt;</span><span class="language-css"></span></span><br><span class="line"><span class="language-css">    <span class="selector-class">.container</span> &#123;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">display</span>: flex;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">flex-wrap</span>: wrap;</span></span><br><span class="line"><span class="language-css">    &#125;</span></span><br><span class="line"><span class="language-css"></span></span><br><span class="line"><span class="language-css">    <span class="selector-class">.chart</span> &#123;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">flex-basis</span>: <span class="number">50%</span>;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">width</span>: auto;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">height</span>: <span class="number">400px</span>;</span></span><br><span class="line"><span class="language-css">    &#125;</span></span><br><span class="line"><span class="language-css">  </span><span class="tag">&lt;/<span class="name">style</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">&quot;chart&quot;</span> <span class="attr">class</span>=<span class="string">&quot;chart&quot;</span>&gt;</span><span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Visualization</category>
-      </categories>
-      <tags>
-        <tag>树莓派</tag>
-        <tag>Google Charts</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>CUDA官方文档笔记</title>
-    <url>/hexo_blog/2018/05/18/CUDA/CUDA%E5%AE%98%E6%96%B9%E6%96%87%E6%A1%A3%E7%AC%94%E8%AE%B0/</url>
-    <content><![CDATA[<p>用CUDA实现高效的矩阵运算——从一维到三维向量加法、共享内存矩阵乘法2.0。接
-<a href="/hexo_blog/2018/05/13/CUDA/CUDA%E7%BC%96%E7%A8%8B%E5%85%A5%E9%97%A8/" title="CUDA编程入门">CUDA编程入门</a>。</p>
-<span id="more"></span>
-<p><a
-href="https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html">官方文档</a></p>
-<p>一维向量加法1.0</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 核函数定义</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">VecAdd</span><span class="params">(<span class="type">float</span>* A, <span class="type">float</span>* B, <span class="type">float</span>* C)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    C[i] = A[i] + B[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    ...</span><br><span class="line">    <span class="comment">// 用N个线程调用核函数，N目前最多为1024</span></span><br><span class="line">    VecAdd&lt;&lt;&lt;<span class="number">1</span>, N&gt;&gt;&gt;(A, B, C);</span><br><span class="line">    ...</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>二维向量加法1.0</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 核函数，二维数组</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatAdd</span><span class="params">(<span class="type">float</span> A[N][N], <span class="type">float</span> B[N][N],</span></span><br><span class="line"><span class="params">                       <span class="type">float</span> C[N][N])</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="type">int</span> j = threadIdx.y;</span><br><span class="line">    C[i][j] = A[i][j] + B[i][j];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    ...</span><br><span class="line">    <span class="comment">// 设置调用GPU参数</span></span><br><span class="line">    <span class="comment">// N * N * 1 threads</span></span><br><span class="line">    <span class="comment">// 一个block，里面有N个线程</span></span><br><span class="line">    <span class="type">int</span> numBlocks = <span class="number">1</span>;</span><br><span class="line">    dim3 <span class="title function_">threadsPerBlock</span><span class="params">(N, N)</span>;</span><br><span class="line">    MatAdd&lt;&lt;&lt;numBlocks, threadsPerBlock&gt;&gt;&gt;(A, B, C);</span><br><span class="line">    ...</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>二维向量加法2.0</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">//</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatAdd</span><span class="params">(<span class="type">float</span> A[N][N], <span class="type">float</span> B[N][N],</span></span><br><span class="line"><span class="params"><span class="type">float</span> C[N][N])</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 使用block</span></span><br><span class="line">    <span class="type">int</span> i = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> j = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="keyword">if</span> (i &lt; N &amp;&amp; j &lt; N)</span><br><span class="line">        C[i][j] = A[i][j] + B[i][j];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    ...</span><br><span class="line">    <span class="comment">// 设置线程参数，block参数。线程block一般为16*16</span></span><br><span class="line">    dim3 <span class="title function_">threadsPerBlock</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span>;</span><br><span class="line">    dim3 <span class="title function_">numBlocks</span><span class="params">(N / threadsPerBlock.x, N / threadsPerBlock.y)</span>;</span><br><span class="line">    MatAdd&lt;&lt;&lt;numBlocks, threadsPerBlock&gt;&gt;&gt;(A, B, C);</span><br><span class="line">    ...</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>一维向量2.0</p>
-<p>这里的设备指的是GPU</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 核函数，使用block</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">VecAdd</span><span class="params">(<span class="type">float</span>* A, <span class="type">float</span>* B, <span class="type">float</span>* C, <span class="type">int</span> N)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = blockDim.x * blockIdx.x + threadIdx.x;</span><br><span class="line">    <span class="keyword">if</span> (i &lt; N)</span><br><span class="line">        C[i] = A[i] + B[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 主函数，调用CPU</span></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> N = ...;</span><br><span class="line">    <span class="type">size_t</span> size = N * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请主机指针内存</span></span><br><span class="line">    <span class="type">float</span>* h_A = (<span class="type">float</span>*)<span class="built_in">malloc</span>(size);</span><br><span class="line">    <span class="type">float</span>* h_B = (<span class="type">float</span>*)<span class="built_in">malloc</span>(size);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 初始化一些输入向量（参数）</span></span><br><span class="line">    ...</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请分配设备内存</span></span><br><span class="line">    <span class="type">float</span>* d_A;</span><br><span class="line">    cudaMalloc(&amp;d_A, size);</span><br><span class="line">    <span class="type">float</span>* d_B;</span><br><span class="line">    cudaMalloc(&amp;d_B, size);</span><br><span class="line">    <span class="type">float</span>* d_C;</span><br><span class="line">    cudaMalloc(&amp;d_C, size);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 拷贝主机内存到设备中</span></span><br><span class="line">    cudaMemcpy(d_A, h_A, size, cudaMemcpyHostToDevice);</span><br><span class="line">    cudaMemcpy(d_B, h_B, size, cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 调用内核</span></span><br><span class="line">    <span class="type">int</span> threadsPerBlock = <span class="number">256</span>;</span><br><span class="line">    <span class="type">int</span> blocksPerGrid =</span><br><span class="line">            (N + threadsPerBlock - <span class="number">1</span>) / threadsPerBlock;</span><br><span class="line">    VecAdd&lt;&lt;&lt;blocksPerGrid, threadsPerBlock&gt;&gt;&gt;(d_A, d_B, d_C, N);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 从设备中拷贝的h_C（结果）到主机中</span></span><br><span class="line">    <span class="comment">// h_C（结果）在主机内存中包含结果</span></span><br><span class="line">    cudaMemcpy(h_C, d_C, size, cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 释放设备内存</span></span><br><span class="line">    cudaFree(d_A);</span><br><span class="line">    cudaFree(d_B);</span><br><span class="line">    cudaFree(d_C);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 释放主机内存</span></span><br><span class="line">    ...</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>二维和三维数组分别需要使用 <code>cudaMallocPitch()</code>和
-<code>cudaMalloc3D()</code></p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 二维数组</span></span><br><span class="line"><span class="comment">// Host code</span></span><br><span class="line"><span class="type">int</span> width = <span class="number">64</span>, height = <span class="number">64</span>;</span><br><span class="line"><span class="type">float</span>* devPtr;</span><br><span class="line"><span class="type">size_t</span> pitch;</span><br><span class="line">cudaMallocPitch(&amp;devPtr, &amp;pitch,</span><br><span class="line">                width * <span class="keyword">sizeof</span>(<span class="type">float</span>), height);</span><br><span class="line">MyKernel&lt;&lt;&lt;<span class="number">100</span>, <span class="number">512</span>&gt;&gt;&gt;(devPtr, pitch, width, height);</span><br><span class="line"></span><br><span class="line"><span class="comment">// Device code</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MyKernel</span><span class="params">(<span class="type">float</span>* devPtr,</span></span><br><span class="line"><span class="params">                         <span class="type">size_t</span> pitch, <span class="type">int</span> width, <span class="type">int</span> height)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> r = <span class="number">0</span>; r &lt; height; ++r) &#123;</span><br><span class="line">        <span class="type">float</span>* row = (<span class="type">float</span>*)((<span class="type">char</span>*)devPtr + r * pitch);</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> c = <span class="number">0</span>; c &lt; width; ++c) &#123;</span><br><span class="line">            <span class="type">float</span> element = row[c];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 三维数组</span></span><br><span class="line"><span class="comment">// Host code</span></span><br><span class="line"><span class="type">int</span> width = <span class="number">64</span>, height = <span class="number">64</span>, depth = <span class="number">64</span>;</span><br><span class="line">cudaExtent extent = make_cudaExtent(width * <span class="keyword">sizeof</span>(<span class="type">float</span>),</span><br><span class="line">                                    height, depth);</span><br><span class="line">cudaPitchedPtr devPitchedPtr;</span><br><span class="line">cudaMalloc3D(&amp;devPitchedPtr, extent);</span><br><span class="line">MyKernel&lt;&lt;&lt;<span class="number">100</span>, <span class="number">512</span>&gt;&gt;&gt;(devPitchedPtr, width, height, depth);</span><br><span class="line"></span><br><span class="line"><span class="comment">// Device code</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MyKernel</span><span class="params">(cudaPitchedPtr devPitchedPtr,</span></span><br><span class="line"><span class="params">                         <span class="type">int</span> width, <span class="type">int</span> height, <span class="type">int</span> depth)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">char</span>* devPtr = devPitchedPtr.ptr;</span><br><span class="line">    <span class="type">size_t</span> pitch = devPitchedPtr.pitch;</span><br><span class="line">    <span class="type">size_t</span> slicePitch = pitch * height;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> z = <span class="number">0</span>; z &lt; depth; ++z) &#123;</span><br><span class="line">        <span class="type">char</span>* slice = devPtr + z * slicePitch;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> y = <span class="number">0</span>; y &lt; height; ++y) &#123;</span><br><span class="line">            <span class="type">float</span>* row = (<span class="type">float</span>*)(slice + y * pitch);</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> x = <span class="number">0</span>; x &lt; width; ++x) &#123;</span><br><span class="line">                <span class="type">float</span> element = row[x];</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 二维、三维数组获取全局变量的方法</span></span><br><span class="line">__constant__ <span class="type">float</span> constData[<span class="number">256</span>];</span><br><span class="line"><span class="type">float</span> data[<span class="number">256</span>];</span><br><span class="line">cudaMemcpyToSymbol(constData, data, <span class="keyword">sizeof</span>(data));</span><br><span class="line">cudaMemcpyFromSymbol(data, constData, <span class="keyword">sizeof</span>(data));</span><br><span class="line"></span><br><span class="line">__device__ <span class="type">float</span> devData;</span><br><span class="line"><span class="type">float</span> value = <span class="number">3.14f</span>;</span><br><span class="line">cudaMemcpyToSymbol(devData, &amp;value, <span class="keyword">sizeof</span>(<span class="type">float</span>));</span><br><span class="line"></span><br><span class="line">__device__ <span class="type">float</span>* devPointer;</span><br><span class="line"><span class="type">float</span>* ptr;</span><br><span class="line">cudaMalloc(&amp;ptr, <span class="number">256</span> * <span class="keyword">sizeof</span>(<span class="type">float</span>));</span><br><span class="line">cudaMemcpyToSymbol(devPointer, &amp;ptr, <span class="keyword">sizeof</span>(ptr));</span><br></pre></td></tr></table></figure>
-<p>共享内存——矩阵乘法1.0</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 定义一个矩阵的数据结构</span></span><br><span class="line"><span class="comment">// Matrices are stored in row-major order:</span></span><br><span class="line"><span class="comment">// M(row, col) = *(M.elements + row * M.width + col)</span></span><br><span class="line"><span class="keyword">typedef</span> <span class="class"><span class="keyword">struct</span> &#123;</span></span><br><span class="line">    <span class="type">int</span> width;</span><br><span class="line">    <span class="type">int</span> height;</span><br><span class="line">    <span class="type">float</span>* elements;</span><br><span class="line">&#125; Matrix;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 线程锁大小</span></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> BLOCK_SIZE 16</span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 提前声明矩阵乘法核函数</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(<span class="type">const</span> Matrix, <span class="type">const</span> Matrix, Matrix)</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 矩阵乘法主机代码</span></span><br><span class="line"><span class="comment">// 矩阵维度设为线程锁大小的倍数</span></span><br><span class="line"><span class="type">void</span> <span class="title function_">MatMul</span><span class="params">(<span class="type">const</span> Matrix A, <span class="type">const</span> Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// Load A and B to device memory</span></span><br><span class="line">    Matrix d_A;</span><br><span class="line">    d_A.width = A.width; d_A.height = A.height;</span><br><span class="line">    <span class="type">size_t</span> size = A.width * A.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_A.elements, size);</span><br><span class="line">    cudaMemcpy(d_A.elements, A.elements, size,</span><br><span class="line">               cudaMemcpyHostToDevice);</span><br><span class="line">    Matrix d_B;</span><br><span class="line">    d_B.width = B.width; d_B.height = B.height;</span><br><span class="line">    size = B.width * B.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_B.elements, size);</span><br><span class="line">    cudaMemcpy(d_B.elements, B.elements, size,</span><br><span class="line">               cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Allocate C in device memory</span></span><br><span class="line">    Matrix d_C;</span><br><span class="line">    d_C.width = C.width; d_C.height = C.height;</span><br><span class="line">    size = C.width * C.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_C.elements, size);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Invoke kernel</span></span><br><span class="line">    dim3 <span class="title function_">dimBlock</span><span class="params">(BLOCK_SIZE, BLOCK_SIZE)</span>;</span><br><span class="line"></span><br><span class="line">    dim3 <span class="title function_">dimGrid</span><span class="params">(B.width / dimBlock.x, A.height / dimBlock.y)</span>;</span><br><span class="line">    MatMulKernel&lt;&lt;&lt;dimGrid, dimBlock&gt;&gt;&gt;(d_A, d_B, d_C);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Read C from device memory</span></span><br><span class="line">    cudaMemcpy(C.elements, d_C.elements, size,</span><br><span class="line">               cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Free device memory</span></span><br><span class="line">    cudaFree(d_A.elements);</span><br><span class="line">    cudaFree(d_B.elements);</span><br><span class="line">    cudaFree(d_C.elements);</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiplication kernel called by MatMul()</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(Matrix A, Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// Each thread computes one element of C</span></span><br><span class="line">    <span class="comment">// by accumulating results into Cvalue</span></span><br><span class="line">    <span class="type">float</span> Cvalue = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> e = <span class="number">0</span>; e &lt; A.width; ++e)</span><br><span class="line">        Cvalue += A.elements[row * A.width + e]</span><br><span class="line">                * B.elements[e * B.width + col];</span><br><span class="line">    C.elements[row * C.width + col] = Cvalue;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>矩阵乘法2.0</p>
-<p>使用 <code>__device__</code>获取矩阵中元素和设定元素</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// Matrices are stored in row-major order:</span></span><br><span class="line"><span class="comment">// M(row, col) = *(M.elements + row * M.stride + col)</span></span><br><span class="line"><span class="keyword">typedef</span> <span class="class"><span class="keyword">struct</span> &#123;</span></span><br><span class="line">    <span class="type">int</span> width;</span><br><span class="line">    <span class="type">int</span> height;</span><br><span class="line">    <span class="type">int</span> stride;</span><br><span class="line">    <span class="type">float</span>* elements;</span><br><span class="line">&#125; Matrix;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 获取矩阵元素</span></span><br><span class="line">__device__ <span class="type">float</span> <span class="title function_">GetElement</span><span class="params">(<span class="type">const</span> Matrix A, <span class="type">int</span> row, <span class="type">int</span> col)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="keyword">return</span> A.elements[row * A.stride + col];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 设定矩阵元素</span></span><br><span class="line">__device__ <span class="type">void</span> <span class="title function_">SetElement</span><span class="params">(Matrix A, <span class="type">int</span> row, <span class="type">int</span> col,</span></span><br><span class="line"><span class="params">                           <span class="type">float</span> value)</span></span><br><span class="line">&#123;</span><br><span class="line">    A.elements[row * A.stride + col] = value;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 根据线程锁的大小构建给定矩阵的子矩阵</span></span><br><span class="line"> __device__ Matrix <span class="title function_">GetSubMatrix</span><span class="params">(Matrix A, <span class="type">int</span> row, <span class="type">int</span> col)</span></span><br><span class="line">&#123;</span><br><span class="line">    Matrix Asub;</span><br><span class="line">    Asub.width    = BLOCK_SIZE;</span><br><span class="line">    Asub.height   = BLOCK_SIZE;</span><br><span class="line">    Asub.stride   = A.stride;</span><br><span class="line">    Asub.elements = &amp;A.elements[A.stride * BLOCK_SIZE * row</span><br><span class="line">                                         + BLOCK_SIZE * col];</span><br><span class="line">    <span class="keyword">return</span> Asub;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> BLOCK_SIZE 16</span></span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(<span class="type">const</span> Matrix, <span class="type">const</span> Matrix, Matrix)</span>;</span><br><span class="line"></span><br><span class="line"><span class="type">void</span> <span class="title function_">MatMul</span><span class="params">(<span class="type">const</span> Matrix A, <span class="type">const</span> Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line"></span><br><span class="line">    Matrix d_A;</span><br><span class="line">    d_A.width = d_A.stride = A.width; d_A.height = A.height;</span><br><span class="line">    <span class="type">size_t</span> size = A.width * A.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_A.elements, size);</span><br><span class="line">    cudaMemcpy(d_A.elements, A.elements, size,</span><br><span class="line">               cudaMemcpyHostToDevice);</span><br><span class="line">    Matrix d_B;</span><br><span class="line">    d_B.width = d_B.stride = B.width; d_B.height = B.height;</span><br><span class="line">    size = B.width * B.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_B.elements, size);</span><br><span class="line">    cudaMemcpy(d_B.elements, B.elements, size,</span><br><span class="line">    cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    Matrix d_C;</span><br><span class="line">    d_C.width = d_C.stride = C.width; d_C.height = C.height;</span><br><span class="line">    size = C.width * C.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_C.elements, size);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Invoke kernel</span></span><br><span class="line">    dim3 <span class="title function_">dimBlock</span><span class="params">(BLOCK_SIZE, BLOCK_SIZE)</span>;</span><br><span class="line">    dim3 <span class="title function_">dimGrid</span><span class="params">(B.width / dimBlock.x, A.height / dimBlock.y)</span>;</span><br><span class="line">    MatMulKernel&lt;&lt;&lt;dimGrid, dimBlock&gt;&gt;&gt;(d_A, d_B, d_C);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Read C from device memory</span></span><br><span class="line">    cudaMemcpy(C.elements, d_C.elements, size,</span><br><span class="line">               cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Free device memory</span></span><br><span class="line">    cudaFree(d_A.elements);</span><br><span class="line">    cudaFree(d_B.elements);</span><br><span class="line">    cudaFree(d_C.elements);</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiplication kernel called by MatMul()</span></span><br><span class="line"> __global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(Matrix A, Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 定义块的行与列</span></span><br><span class="line">    <span class="type">int</span> blockRow = blockIdx.y;</span><br><span class="line">    <span class="type">int</span> blockCol = blockIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 每个线程块计算每一个子矩阵</span></span><br><span class="line">    Matrix Csub = GetSubMatrix(C, blockRow, blockCol);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 每个线程计算每个矩阵的元素，结果保留在Cvalue中</span></span><br><span class="line">    <span class="type">float</span> Cvalue = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 定义Csub中线程的行列</span></span><br><span class="line">    <span class="type">int</span> row = threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 对矩阵A、B的所有子矩阵进行循环</span></span><br><span class="line">    <span class="comment">// 计算Csub，把每一对子矩阵Csub循环，之后累加求和</span></span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> m = <span class="number">0</span>; m &lt; (A.width / BLOCK_SIZE); ++m) &#123;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 获取矩阵A、B的子矩阵</span></span><br><span class="line">        Matrix Asub = GetSubMatrix(A, blockRow, m);</span><br><span class="line">        Matrix Bsub = GetSubMatrix(B, m, blockCol);</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 存储A、B矩阵的子矩阵Asub和Bsub用于共享</span></span><br><span class="line">        __shared__ <span class="type">float</span> As[BLOCK_SIZE][BLOCK_SIZE];</span><br><span class="line">        __shared__ <span class="type">float</span> Bs[BLOCK_SIZE][BLOCK_SIZE];</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 从设备中载入Asub与Bsub到共享内存中</span></span><br><span class="line">        <span class="comment">// 每个线程加载每个子矩阵的一个元素</span></span><br><span class="line">        As[row][col] = GetElement(Asub, row, col);</span><br><span class="line">        Bs[row][col] = GetElement(Bsub, row, col);</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 在开始计算前同步以确保子矩阵被加载</span></span><br><span class="line">        __syncthreads();</span><br><span class="line">        <span class="comment">// 同时计算A、B子矩阵</span></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> e = <span class="number">0</span>; e &lt; BLOCK_SIZE; ++e)</span><br><span class="line">            Cvalue += As[row][e] * Bs[e][col];</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 同步以确保A、B子矩阵在下次迭代前计算完成</span></span><br><span class="line">        __syncthreads();</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 写入C子矩阵到设备内存中，每个线程写入一个元素</span></span><br><span class="line">    SetElement(Csub, row, col, Cvalue);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>CUDA</category>
-      </categories>
-      <tags>
-        <tag>GPU</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>CUDA编程二次入门</title>
-    <url>/hexo_blog/2018/05/18/CUDA/CUDA%E7%BC%96%E7%A8%8B%E4%BA%8C%E6%AC%A1%E5%85%A5%E9%97%A8/</url>
-    <content><![CDATA[<p>本文主要介绍了基本的CUDA程序框架及代码解析，包括如何申请设备指针内存、将数据拷贝到设备上、调用核函数进行计算、线程同步以及将结果拷贝回主机内等。同时也介绍了一些常见问题及解决方案，比如CUDA程序黑屏之后恢复的问题。接
-<a href="/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/" title="记被CUDA折腾死去活来的那十天">记被CUDA折腾死去活来的那十天</a> 。阅读本文前需要有一部分C语言基础。</p>
-<span id="more"></span>
-<h2 id="原始代码">原始代码</h2>
-<p>在VS2015中，新建一个CUDA项目，里面会有一个
-<code>kernel.cu</code>的文件。里面的代码就是官方给出的实例代码。如下所示。</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>,</span><br><span class="line">        c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h2 id="拆解原始代码">拆解原始代码</h2>
-<p>下面对上面的代码逐一进行拆解进行说明。最原始最简单的代码应该是下面这样。在配置正确的情况下，下面的代码应该也是可以正确运行并输出的。</p>
-<p>设备指的是GPU设备</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 引入相关模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 调用核函数，利用gpu进行计算</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="comment">// 获取当前线程id</span></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="comment">// 在同一个线程id下，a数组与b数组元素对应相加</span></span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line">    <span class="comment">// 一般c语言的语法，在这里不做介绍</span></span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请设备指针，每个变量对应一个设备指针。所以这里有三个</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 选择GPU设备，如果只有一个就只需选择0</span></span><br><span class="line">    cudaSetDevice(<span class="number">0</span>);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请设备指针的内存</span></span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将主机上的a、b数组的数据拷贝到设备上。</span></span><br><span class="line">    cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 调用核函数，注意这里调用函数的语法与一般的不同。&lt;&lt;&lt;1, size&gt;&gt;&gt;指的是分配一个线程组，里面有size个线程</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 进行线程同步</span></span><br><span class="line">    cudaThreadSynchronize();</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将结果拷贝回主机里的c变量中</span></span><br><span class="line">    cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 释放设备指针内存</span></span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 输出结果</span></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>, c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 退出线程</span></span><br><span class="line">    cudaThreadExit();</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p><code>kernel.cu</code>里面的代码是基于上面这个代码进行拓展。</p>
-<ol type="1">
-<li>每一次操作之后，进行判断状态
-<code>cudaStatus</code>，如果状态异常就退出，这是一种比较好的编程习惯，当然带来的影响就是可读性略微降低了。</li>
-<li>封装了一个函数，将操作主机的代码与操作设备的代码（比如申请设备指针内存与释放设备指针等）。</li>
-</ol>
-<h2 id="拓展代码">拓展代码</h2>
-<p>当然，我们还可以基于此再次进行拓展。下面介绍几个比较常用的拓展代码。</p>
-<p>待补充。。。。。</p>
-<h2 id="补充说明">补充说明</h2>
-<p>如果在运行cuda代码的时候，显示屏突然黑屏，之后又恢复正常，是因为程序时间运行过长，触发了TDR事件，导致黑屏，显卡运算中断。解决方案如下：</p>
-<ol type="1">
-<li>开始菜单中找到
-<code>Nsight Monitor</code>并打开。（Win10可以直接搜索）</li>
-<li>在任务栏中打开 <code>Nsight Monitor</code>，单击右下角的
-<code>Nsight Monitor options</code>。</li>
-<li>设置 <code>General</code>的 <code>Microsoft Display Driver</code>中
-<code>WDDM TDR Delay</code>中的值，把
-<code>2</code>(默认一般是2)，调大，比如200。</li>
-<li>保存后退出，重启电脑即可。</li>
-</ol>
-<p>详见<a
-href="http://blog.163.com/yuhua_kui/blog/static/9679964420146183211348/">解决CUDA程序的黑屏恢复问题</a></p>
-]]></content>
-      <categories>
-        <category>CUDA</category>
-      </categories>
-      <tags>
-        <tag>GPU</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>CUDA编程入门</title>
-    <url>/hexo_blog/2018/05/13/CUDA/CUDA%E7%BC%96%E7%A8%8B%E5%85%A5%E9%97%A8/</url>
-    <content><![CDATA[<p>CUDA编程入门，接 <a href="/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/" title="记被CUDA折腾死去活来的那十天">记被CUDA折腾死去活来的那十天</a></p>
-<span id="more"></span>
-<p>参考文章：<a
-href="https://blog.csdn.net/augusdi/article/details/12833235">CUDA从入门到精通</a></p>
-<h2 id="注意事项">注意事项</h2>
-<p>远程桌面登陆服务器时，远程终端不能运行CUDA程序。原因是因为远程桌面登陆调用的显卡资源是本地的显卡，运行程序的时候没有远程的显卡，自然会报错。所以可以用远程终端进行登陆，或者远程服务器配置两块显卡，一块用于显示一块用于计算。</p>
-<p>话不多说，直接上代码。</p>
-<h2 id="原始的demo线程并行">原始的Demo（线程并行）</h2>
-<p>目的就是a, b两个数组对应元素进行相加</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 导入cuda相关模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="comment">// c中自带的模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 声明一个函数后面需要调用</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">// __global__前缀表示用于控制设备，还有一个控制设备的前缀是__device__</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 这个函数是运行在GPU上的，称为核函数</span></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// cuda_Error_t指的是cuda错误类型，取值为整数</span></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="comment">// cudaDeviceProp是设备属性结构体</span></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>,</span><br><span class="line">        c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// cudaThreadExit must be called before exiting in order for profiling and</span></span><br><span class="line">    <span class="comment">// tracing tools such as Nsight and Visual Profiler to show complete traces.</span></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 用于使用CUDA并行添加向量的辅助函数</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// GPU设备端的数据指针</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    <span class="comment">// GPU状态指示</span></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 当电脑上有多个显卡支持CUDA时，选择某个GPU来进行运算。0表示设备号，还有1,2,3……</span></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 分配GPU设备端的内存。三个变量，所以需要分配三次。</span></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="comment">// 申请失败就报错</span></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// 拷贝数据到GPU中，已有两个变量，所以需要拷贝两次</span></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 运行核函数。关键代码</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 同步线程</span></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将结果拷贝回主机</span></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    <span class="comment">// 释放GPU设备端内存</span></span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>以上是线程并行，CUDA官方初始给的例子。</p>
-<p>启动核函数的调用过程，即<code>addKernel</code>函数。</p>
-<p>&lt;&lt;&lt;&gt;&gt;&gt;表示运行时配置符号，里面1表示只分配一个线程组（又称线程块、Block），size表示每个线程组有size个线程（Thread）。本程序中size根据前面传递参数个数应该为5，所以运行的时候，核函数在5个GPU线程单元上分别运行了一次，总共运行了5次。这5个线程是如何知道自己“身份”的？是靠<code>threadIdx</code>这个内置变量，它是个dim3类型变量，接受&lt;&lt;&lt;&gt;&gt;&gt;中第二个参数，它包含x,y,z
-3维坐标，而我们传入的参数只有一维，所以只有x值是有效的。通过核函数中<code>int i = threadIdx.x;</code>这一句，每个线程可以获得自身的id号，从而找到自己的任务去执行。</p>
-<h2 id="块并行">块并行</h2>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = blockIdx.x;</span><br><span class="line">	c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">addKernel&lt;&lt;&lt;size,<span class="number">1</span> &gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br></pre></td></tr></table></figure>
-<p>这里与线程并行的区别在于，由<code>threadIdx</code>改为了<code>blockIdx</code>。<code>&lt;&lt;&lt;1, size&gt;&gt;&gt;</code>改为了<code>&lt;&lt;&lt;size, 1&gt;&gt;&gt;</code>。</p>
-<p>线程并行是细粒度并行，调度效率高；块并行是粗粒度并行，每次调度都要重新分配资源，有时资源只有一份，那么所有线程块都只能排成一队，串行执行。</p>
-<p>有时采用分治法，将一个大问题分解为几个小规模问题，将这些小规模问题分别用一个线程块实现，线程块内可以采用细粒度的线程并行，而块之间为粗粒度并行，这样可以充分利用硬件资源，降低线程并行的计算复杂度。适当分解，降低规模，在一些矩阵乘法、向量内积计算应用中可以得到充分的展示。</p>
+<h2 id="resultword-ordering">Result——Word Ordering</h2>
+<p><strong>The influence patterns to the ordering of the
+words.</strong></p>
+<p>可用于验证模型的泛化性</p>
+<h3 id="单词旋转实验">单词旋转实验</h3>
+<p>原始：</p>
+<ul>
+<li>Prompt：The first President of the Republic of Astrobia was</li>
+<li>Completion：Zorald Pfaff</li>
+</ul>
+<p>结论：</p>
+<ul>
+<li>与 Prompt 和 Completion
+相关的短语只要按顺序出现始终能够保持较高的稳定性</li>
+<li>翻转 Prompt 和 Completion 变化较小</li>
+<li>哪怕删除 Prompt，影响也没有改变，最为关键的是 Completion</li>
+</ul>
+<table>
+<colgroup>
+<col style="width: 29%" />
+<col style="width: 70%" />
+</colgroup>
+<thead>
+<tr>
+<th>分别修改 prompt 和 completion</th>
+<th><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/ab33a902e1044c60be66b8c11b443bdd.webp" /></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>删掉 Zorald Pfaff</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5b4cd20d88204baf990d406ba5041ad8.webp" /></td>
+</tr>
+<tr>
+<td>修改 Zorald Pfaff 和 President of the Republic</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/6a37b66b7d3941dcb1db42ce8ca1d073.webp" /></td>
+</tr>
+<tr>
+<td>完全不一致</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/5211b0d5e2c34191be9618f30bc44bca.webp" /></td>
+</tr>
+<tr>
+<td>调换 prompt 和 completion 位置</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e547d3cd16844505a24abb2bab404c8e.webp" /></td>
+</tr>
+<tr>
+<td>调换位置，只保留 Zorald Pfaff</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e9daf868aa0e483b84fb6ef5db21ac9b.webp" /></td>
+</tr>
+</tbody>
+</table>
+<p>大模型相较于小模型，对于单词的变化更加敏感</p>
+<p>注：合成句子不在训练集中</p>
+<p>Maybe相关 <a
+href="https://www.jiqizhixin.com/articles/2023-11-18-5">https://www.jiqizhixin.com/articles/2023-11-18-5</a></p>
+<h3 id="英翻中-中翻英">英翻中-中翻英</h3>
+<p>结论：</p>
+<ul>
+<li>不同模型大小具有相同趋势（口头描述）</li>
+<li>影响至少减少了一个数量级</li>
+<li>甚至只要翻转一个单词顺序，影响分数也会显著降低</li>
+</ul>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/cuda/20130723220559500.png" /></p>
-<p>多个线程块组成一个线程格(Grid)。一维线程-&gt;二维线程块-&gt;三维线程格。</p>
-<h2 id="流并行">流并行</h2>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line">addKernel&lt;&lt;&lt;size, <span class="number">1</span>&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">cudaStream_t stream[<span class="number">5</span>];</span><br><span class="line"><span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">    <span class="comment">// 创建流</span></span><br><span class="line">    cudaStreamCreate(&amp;stream[i]);</span><br><span class="line">&#125;</span><br><span class="line"><span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">	<span class="comment">// 执行流。线程块数(1)，每个线程中线程数(1)，每个block用到的共享内存大小(0)，流对象(当前核函数在哪个流上运行)</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, <span class="number">1</span>, <span class="number">0</span>, stream[i]&gt;&gt;&gt;(dev_c + i, dev_a + i, dev_b + i);</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 控制整个设备的所有流同步</span></span><br><span class="line">cudaDeviceSynchronize();</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">	cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">Error:</span><br><span class="line">	<span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">        <span class="comment">//销毁流</span></span><br><span class="line">        cudaStreamDestroy(stream[i]);</span><br><span class="line">    &#125;</span><br><span class="line">	cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br></pre></td></tr></table></figure>
-<p>利用多个网格（线程格）进行流并行。这样实现的并行可以达到比块并行更粗的细粒度。</p>
-<h2 id="线程通信">线程通信</h2>
-<p>这次目的分三个线程做三个任务，1-5数字进行求和，1-5每个数字求平方和，1-5累乘</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 更改addKernel函数内容如下</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a)</span> &#123;</span><br><span class="line">    <span class="comment">// 线程块数：1，块号为0</span></span><br><span class="line">    <span class="comment">// 线程数：5，线程号0~4</span></span><br><span class="line">    <span class="comment">// 共享存储器大小：5个int型变量大小（5 * sizeof(int）)</span></span><br><span class="line"></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="comment">// 将数据一次读入到SM（Shared Memory）内部</span></span><br><span class="line">    <span class="keyword">extern</span> __shared__ <span class="type">int</span> smem[];</span><br><span class="line">    smem[i] = a[i];</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 0号线程做平方和</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">0</span>) &#123;</span><br><span class="line">        c[<span class="number">0</span>] = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span>(<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">0</span>] += smem[d] * smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 1号线程做累加</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">1</span>) &#123;</span><br><span class="line">        c[<span class="number">1</span>] = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span>(<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">1</span>] += smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 2号线程做累乘</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">2</span>) &#123;</span><br><span class="line">        c[<span class="number">2</span>] = <span class="number">1</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">2</span>] *= smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 更改main函数的内容</span></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line"> 	<span class="comment">// 跟普通的C语言代码编写差不多</span></span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;\t1+2+3+4+5 = %d\n\t1^2+2^2+3^2+4^2+5^2 = %d\n\t1*2*3*4*5 = %d\n\n\n\n\n\n&quot;</span>, c[<span class="number">1</span>], c[<span class="number">0</span>], c[<span class="number">2</span>]);</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 更改addWithCuda内容</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a,  <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 去除c变量相关内容，如申请内存、GPU设备端指针。其余内容差不多</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)&#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 第三个参数为共享内存大小（字节数）</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size, size * <span class="title function_">sizeof</span><span class="params">(<span class="type">int</span>)</span>, 0&gt;&gt;&gt;<span class="params">(dev_c, dev_a)</span>;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h2 id="性能剖析">性能剖析</h2>
-<p>测试对比线程并行和块并行的速度</p>
-<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel_blk</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="type">int</span> i = blockIdx.x;</span><br><span class="line">    c[i] = a[i]+ b[i];</span><br><span class="line">&#125;</span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel_thd</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i]+ b[i];</span><br><span class="line">&#125;</span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">1024</span>;</span><br><span class="line">    <span class="type">int</span> a[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line">    <span class="type">int</span> b[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; arraySize;i++) &#123;</span><br><span class="line">        a[i] = i;</span><br><span class="line">        b[i] = arraySize-i;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line">    <span class="type">int</span> num = <span class="number">0</span>;</span><br><span class="line">    <span class="comment">// 设备属性结构体</span></span><br><span class="line">    cudaDeviceProp prop;</span><br><span class="line">    <span class="comment">// 获取设备总数</span></span><br><span class="line">    cudaStatus = cudaGetDeviceCount(&amp;num);</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i&lt;num;i++) &#123;</span><br><span class="line">        <span class="comment">// 获取设备属性</span></span><br><span class="line">        cudaGetDeviceProperties(&amp;prop,i);</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i&lt;arraySize;i++) &#123;</span><br><span class="line">        <span class="keyword">if</span>(c[i] != (a[i]+b[i])) &#123;</span><br><span class="line">            <span class="built_in">printf</span>(<span class="string">&quot;Error in %d\n&quot;</span>,i);</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span> &#123;</span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// Copy input vectors from host memory to GPU buffers.</span></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 计算核函数执行时间</span></span><br><span class="line">    cudaEvent_t start, stop;</span><br><span class="line">    cudaEventCreate(&amp;start);</span><br><span class="line">    cudaEventCreate(&amp;stop);</span><br><span class="line">    cudaEventRecord(start, <span class="number">0</span>);</span><br><span class="line">    <span class="comment">// 由于数据量小，执行1000次扩大执行时间。blk是块并行，thd是线程并行</span></span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">1000</span>;i++) &#123;</span><br><span class="line"><span class="comment">//      addKernel_blk&lt;&lt;&lt;size,1&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span></span><br><span class="line">        addKernel_thd&lt;&lt;&lt;<span class="number">1</span>,size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line">    &#125;</span><br><span class="line">    cudaEventRecord(stop,<span class="number">0</span>);</span><br><span class="line">    cudaEventSynchronize(stop);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// stop-start，执行总时间</span></span><br><span class="line">    <span class="type">float</span> tm;</span><br><span class="line">    cudaEventElapsedTime(&amp;tm, start, stop);</span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;GPU Elapsed time:%.6f ms.\n&quot;</span>, tm);</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// cudaThreadSynchronize waits for the kernel to finish, and returns</span></span><br><span class="line">    <span class="comment">// any errors encountered during the launch.</span></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// Copy output vector from GPU buffer to host memory.</span></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>CUDA</category>
-      </categories>
-      <tags>
-        <tag>GPU</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>记被CUDA折腾死去活来的那十天</title>
-    <url>/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/</url>
-    <content><![CDATA[<p>谨以此文章纪念大二的一次折腾CUDA环境搭建经历。</p>
-<span id="more"></span>
-<p>为了提高矩阵运算的速度和不浪费实验室的GPU，所以开始折腾CUDA环境，入门GPU
-Coder。感谢@小岳岳 全程的帮助，学习了很多硬件知识；感谢@浪浪
-学长的指导，少走了一些弯路。本文会尽量从零开始介绍硬件知识过渡到系统和软件环境的搭建。</p>
-<p>贴上学长的博客<a href="http://amourll.cn/2018/03/27/gpu-coder/">GPU
-Coder初体验</a></p>
-<h2 id="基础介绍">基础介绍</h2>
-<p>首先祭出显卡天梯图，时间截止于博客编辑完的日期（2018年4月）。</p>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/877f0424698540ffa4fc03178caf36f5.webp" /></p>
+<p>英语翻译为汉语，始终比汉语翻译为英语（手动构造）具有更高的影响力</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/cuda/5af851eca8de0.jpg" /></p>
-<p>关于显卡的一些基本知识可以看知乎的这个贴：<a
-href="https://zhuanlan.zhihu.com/p/33743591">知识扫盲I给大众的电脑显卡选购指南</a></p>
-<p>这里再简单补充一些基础知识，GPU（Graphic Process Unit
-）其实是显卡的核心部件，分为AMD和Nvidia两种。一般情况下，做科学计算或者深度学习都是用Nvidia显卡，当然用AMD的也有在这里我用的是Nvidia所以就不介绍如何搭建AMD的编程环境了。</p>
-<h3 id="关于gpu编程">关于GPU编程</h3>
-<p>所谓GPU编程，我的理解是用代码操作GPU进行运算。通常来说，我们写代码用软件都是调用CPU来工作的，显卡在常规的认知中（某某年以前）都是用来作为图像渲染、玩游戏的。随着时代的发展，关于图像处理的数据越来越多，交给CPU来计算就显得过慢，所以人们开始用起了GPU，GPU原本发明的用途就是为了处理图像。所以如果数据是关于图像处理的，交给GPU来计算远远好于CPU的。</p>
-<h3 id="关于并行计算">关于并行计算</h3>
-<p>GPU并行：CUDA，OpenCL（都比较麻烦，建议CUDA）
-CPU并行：MPI，OpenMP。MPI比较麻烦，OpenMP容易实现</p>
-<p>OpenAcc：OpenMP和CUDA的混合</p>
-<p>这里使用的方案是CUDA（读作：库达）。</p>
-<h3 id="实验室的显卡介绍">实验室的显卡介绍</h3>
-<p>Tesla C2050、两块Titans
-X、一块2010/2011年的AMD卡（具体忘了，反正只用来做图像显示）</p>
-<p>配置情况：特斯拉+AMD，AMD用做显示，特斯拉用来计算。至于为什么不上泰坦，是因为要预备留着后面买的新服务器，这台是之前（2010/2011）的服务器。简单说说特斯拉显卡，特斯拉显示是英伟达专门生产用作科学计算的，一般情况下特斯拉显卡的价格和计算性能都可以碾压其他类型的显卡，而且支持双精度。不过在这的话，泰坦还是可以吊打这块特斯拉的，唯一的缺点大概泰坦是不支持双精度吧。</p>
-<h2 id="正文">正文</h2>
-<h3 id="ubuntu-tesla-amd">Ubuntu + Tesla + AMD</h3>
-<h4 id="结果">结果</h4>
-<p>安装失败</p>
-<h4 id="遇到问题及解决方案">遇到问题及解决方案</h4>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/785f474cd19c4adcb9368f857680f9cb.webp" /></p>
+<p>结论：</p>
+<ul>
+<li>The model has not successfully transferred knowledge of the relation
+itself.</li>
+</ul>
+<p>解释：</p>
 <ol type="1">
-<li>特斯拉不能作为图像显示。解决方案：换AMD显卡做显示。。。</li>
-<li>Ubuntu16.04之后不支持AMD显卡驱动。解决方案：老版本的Ubuntu或者搜索关键词<code>amdgpu-pro</code>、<code>Ubuntu</code></li>
-<li>安装CUDA是否还需要安装英伟达的驱动。Windows下是不需要的，Ubuntu不知道，很多博客中都有安装英伟达驱动这一步骤。</li>
-<li>安装完英伟达驱动之后，出现循环登陆或者不能显示图形界面的情况。解决方案：卸载英伟达驱动，重启。</li>
-<li>英伟达（特斯拉）和AMD双显卡的问题。这种情况网上搜出来的结果比较少，大多都是核显或集显+英伟达显卡。所以在尝试了N次之后，放弃了。</li>
-</ol>
-<h4 id="补充">补充</h4>
-<p>当时不知道装CUDA是否需要安装英伟达驱动，所以就安装了。但是安装完之后重启，就不能显示图形界面，原因如问题一。很矛盾的点就在于，如果只安装CUDA不装英伟达驱动，安装CUDA是失败的。综上，不懂如何在AMD的基础上用特斯拉搭建环境。（最后可能怀疑是显卡本身出了问题）</p>
-<p><a
-href="https://developer.nvidia.com/cuda-downloads">CUDA下载地址</a>，这里选择好系统环境，强烈推荐下载<code>runfile</code>，用它进行安装。不推荐安装最新版本。CUDA官网虽声明说CUDA向下兼容，但根据网上反馈和自己的实验来看，CUDA并没有做到。关于主板BIOS设置显卡的问题，实验室这块并不支持。。。。凉凉</p>
-<h4 id="一些实用命令">一些实用命令</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在不能进入图形界面的时候，按住Ctrl+Alt+F1/F2/F3进入终端界面，输入下面的命令卸载英伟达驱动。卸载完重启即可</span></span><br><span class="line">$ <span class="built_in">sudo</span> apt-get remove --purge nvidia-*</span><br><span class="line">$ reboot</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看显卡信息</span></span><br><span class="line">$ lspci | grep -i vga</span><br><span class="line"><span class="comment"># 查看GPU型号</span></span><br><span class="line">$ lspci | grep -i nvidia</span><br><span class="line"><span class="comment"># 查看NVIDIA驱动版本</span></span><br><span class="line">$ <span class="built_in">sudo</span> dpkg --list | grep nvidia-*</span><br></pre></td></tr></table></figure>
-<p>可以参考的一些博客如下：</p>
-<p><a
-href="https://blog.csdn.net/chaihuimin/article/details/71006654?locationNum=2&amp;fps=1">Ubuntu16.04安装NVIDIA显卡驱动和CUDA时的一些坑与解决方案</a></p>
-<p><a href="http://www.cnblogs.com/sp-li/p/7680526.html">ubuntu16.04
-安装NVIDIA和CUDA8.0</a></p>
-<p><a
-href="https://blog.csdn.net/wonengguwozai/article/details/52664597">Ubuntu安装和NVIDIA驱动和安装</a></p>
-<p><a
-href="https://jingyan.baidu.com/article/d8072ac468a91dec95cefdf0.html">主板设置独显/集显</a></p>
-<p>以上，来来回回重装过无数遍系统，耗时5-6天。其中，大部分耗费的时间在于下载和安装，当然还有相当一部分在重装系统的时间上，而且一开始装的是最新的17.10，之后又回退到了16.04，做了几次系统盘重装了无数次系统，并且还只能在课余时间（基本是晚上，和岳岳在实验室折腾）。可以说是相当惨了，后续已经基本熟练到，出了问题直接手敲命令重新来一遍，不需要查资料（熟悉到令人心疼）。</p>
-<h4 id="总结经验教训">总结经验教训</h4>
-<ol type="1">
-<li>踩坑之前，先去请教已经踩过坑的朋友或者去官方论坛查看安装记录或者搜索相关博客。已经两年了，我的习惯还没有改过来QAQ，喜欢自己一遍又一遍踩坑之后再搜索</li>
-<li>提前准备一些工具，比如如果是大型工具，就可能会弄坏（“脏”）系统，备一个系统盘。还有一些能提前获取的压缩包或者安装包。</li>
-<li>踩坑的时候要脚踏实地，不要浮躁冲动。明明不需要重装系统，浪费时间，就不要直接动手格盘重装</li>
-</ol>
-<h3 id="windows10-tesla-amd">Windows10 + Tesla + AMD</h3>
-<p>按照小岳岳的意思，Windows装这种双显卡驱动啥的会比Linux方便很多，所以改战Windows，重装的系统版本是专业版。</p>
-<h4 id="结果-1">结果</h4>
-<p>失败</p>
-<h4 id="遇到的问题">遇到的问题</h4>
-<p>AMD驱动能够正常显示图形并且可以正常安装驱动，但是那块特斯拉显卡不行。系统能识别出来有两块显卡，但是在安装完英伟达驱动之后，并不能设置特斯拉。且两块显卡，AMD显示正常，特斯拉有个黄色的感叹号。所以是否怀疑是否是显卡本身有了问题（毕竟这么久了QAQ）</p>
-<h4 id="补充-1">补充</h4>
-<p>特斯拉显卡本身不是安装一般的英伟达驱动的，而是需要安装<code>TCC</code>驱动程序，<a
-href="http://www.nvidia.cn/object/software-for-tesla-products-cn.html">官网介绍</a>。这里一开始我是不知道的，所以可能是这里出了问题。但是很矛盾的点在于，CUDA在Windows可以直接安装驱动程序。我也试过直接安装CUDA也不行。浪浪的意思是，特斯拉只能在Server版本上运行，我和小岳岳对这一点持怀疑态度。</p>
-<p>关于完整的安装过程就不在这个失败部分介绍了，下一节完整的说明Windows下安装过程及注意事项。此部分耗时2-3天。</p>
-<h3 id="windows10-titan-x">Windows10 + Titan X</h3>
-<p>由于以上种种原因，并且已经快折腾不下去了，所以跟教授直接申请换泰坦。嗯，所以这一波一天半大概就成功了，只失败了一次。泰坦是支持图形显示的，为了减少问题所以也将AMD换下了（第一次没卸，出了一点问题，可能跟AMD有关）。</p>
-<h4 id="软件版本">软件版本</h4>
-<p>Windows10专业版+VS2013专业版+CUDA7.5</p>
-<p>这里需要注意的是，在前文中提到CUDA不向下兼容，所以推荐不安装最新的，保险起见就选了较旧的版本。关于VS版本选择问题，这里CUDA7.5网上最多的搭配是VS2013，但是这里需要说明VS2013必须要是专业版，否则不支持CUDA。在VS2015之后的版本就没有这个问题。还有就是顺序问题，<strong>一定要先安装VS再安装CUDA</strong>，不需要安装英伟达驱动。</p>
-<h4 id="总体流程">总体流程</h4>
-<ol type="1">
-<li><p>安装Windows系统，激活系统，最好不进行安全更新。卸载一些不必要的软件防止占用空间大小（比如说我只有一块128G的固态，惨！）</p></li>
-<li><p>安装VS2013专业版</p></li>
-<li><p>安装CUDA7.5，建议local版。如果这里安装失败可以进入<strong>服务</strong>，找到<strong>Windows
-Installer</strong>，手动启动它即可。还有一些其他问题，在文末贴了链接。</p></li>
-<li><p>写CUDA的环境配置。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># cuda7.5安装完成之后在系统环境变量中自动配置了两个系统变量</span></span><br><span class="line">CUDA_PATH：C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5</span><br><span class="line">CUDA_PATH_V7_5：C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5</span><br><span class="line"><span class="comment"># 但是为了之后的vs2013的配置做准备我们需要在配置五个系统变量，请根据具体的安装路径进行设置CUDA_SDK_PATH, 这里我用的是默认设置</span></span><br><span class="line">CUDA_BIN_PATH：%CUDA_PATH%\bin</span><br><span class="line">CUDA_LIB_PATH：%CUDA_PATH%\lib\Win32</span><br><span class="line">CUDA_SDK_BIN：%CUDA_SDK_PATH%\bin\Win64</span><br><span class="line">CUDA_SDK_LIB：%CUDA_SDK_PATH%\common\lib\x64</span><br><span class="line">CUDA_SDK_PATH：C:\ProgramData\NVIDIA Corporation\CUDA Samples\v7.5</span><br><span class="line"><span class="comment"># 在系统环境变量path后添加如下内容</span></span><br><span class="line">%CUDA_LIB_PATH%</span><br><span class="line">%CUDA_BIN_PATH%</span><br><span class="line">%CUDA_SDK_LIB_PATH%</span><br><span class="line">%CUDA_SDK_BIN_PATH%</span><br></pre></td></tr></table></figure></li>
-<li><p>进行测试</p></li>
+<li>模型会从 Lower Layers 开始处理已知的 token
+序列，然后逐步升级到抽象层次的表征。但是，对于需要
+Completion，模型必须利用 Upper Layers 进行预测。</li>
+<li>所以 “The first President of the United States was” 是在 Lower
+Layers 的表征，而 “George Washington” 是在 Upper Layers 的表征。</li>
+<li>当 “George Washington” 出现在开头时，那么就会将它放到 Lower Layers
+进行表征，“The first President of the United States” 就是在 Upper
+Layers</li>
+<li>因此，对于模型 Lower Layers 进行更新，并不会影响 Upper Layers
+的表征。（所以 Prompt 删掉后影响分数变化不大）</li>
 </ol>
-<h4 id="参考文章">参考文章</h4>
-<p><a
-href="https://blog.csdn.net/u011821462/article/details/50145221">win10+vs2013+cuda7.5环境搭建</a></p>
-<p><a
-href="https://blog.csdn.net/u012033124/article/details/52169823">VS2013下CUDA
-7.5安装</a></p>
-<p>CUDA安装的一些错误可以参考<a
-href="https://blog.csdn.net/xuxiatian/article/details/50577960">这篇文章</a>，但是在某些情况下，可能系统环境已经被污染了，最好的方法还是重装系统。</p>
-<p>耗时1-2天，总归是很愉快的结束了整个环境搭建。在某些情况下，有些环境的搭建不一定需要（最好不要）用最新的版本，可能会有一些bug，而且之前踩坑的人比较少可借鉴的东西较少。总体来说，抛开之前很奇怪的硬件配置之外，安装过程还是并不难的。接下来，便开始了我的CUDA编程之旅。</p>
-<p>人生路漫漫，谨以此文章纪念上大学来折腾环境最坑爹的一次记录。</p>
-]]></content>
-      <categories>
-        <category>Environment</category>
-      </categories>
-      <tags>
-        <tag>GPU</tag>
-        <tag>CUDA</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Matplotlib笔记</title>
-    <url>/hexo_blog/2017/08/26/PythonPackages/Matplotlib%E7%AC%94%E8%AE%B0/</url>
-    <content><![CDATA[<p>解决Python中文乱码问题的完整方案及Matplotlib绘图基础操作</p>
-<span id="more"></span>
-<h1 id="中文乱码解决方案">中文乱码解决方案</h1>
-<p>支持所有平台的解决方案</p>
-<h2 id="定位默认字体位置">定位默认字体位置</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先进入python的交互环境</span></span><br><span class="line">&gt;&gt;&gt;python</span><br><span class="line"></span><br><span class="line">&gt;from matplotlib.font_manager import findfont, FontProperties</span><br><span class="line"><span class="comment"># 当前使用的默认字体</span></span><br><span class="line">&gt;findfont(FontProperties(family=FontProperties().get_family()))</span><br><span class="line"><span class="comment"># 在此记住返回值</span></span><br></pre></td></tr></table></figure>
-<h2 id="下载支持中文的字体">下载支持中文的字体</h2>
-<p>在这里推荐微软雅黑，msyh.ttf。</p>
-<p><a
-href="http://www.monmonkey.com/sonota/font1/getfont.html">推荐链接</a></p>
-<h2 id="偷梁换柱">偷梁换柱</h2>
-<p>在刚刚查询的那个文件夹中，将原来的默认字体重命名，之后将我们下载msyh.ttf替换进去，并重命名为那个默认字体之前的名字</p>
-<h2 id="大功告成">大功告成</h2>
-<p>当然为了保险起见，还可以操作下面一步。到现在为止，可以测试看看是否成功输出中文。</p>
-<p>找到matplotlib的安装目录，一般是在python的安装目录下的-packages-data,之后修改matplotlibrc文件</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 将下面这两行冒号后面的字符替换</span></span><br><span class="line">...</span><br><span class="line"><span class="comment">#font.sans-serif     : nothing</span></span><br><span class="line">...</span><br><span class="line"><span class="comment">#verbose.level  : debug</span></span><br></pre></td></tr></table></figure>
-<h1 id="matplotlib.pyplot-基础操作">matplotlib.pyplot 基础操作</h1>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> matplotlib.pyplot <span class="keyword">as</span> plt</span><br><span class="line"></span><br><span class="line">x_list = []</span><br><span class="line">y_list = []</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定画布</span></span><br><span class="line">plt.figure(figsize=(<span class="number">8</span>, <span class="number">6</span>), dpi=<span class="number">80</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 画y_list的折线图，格式为&#x27;ro-&#x27;，颜色为blue</span></span><br><span class="line">plt.plot(y_list, <span class="string">&#x27;ro-&#x27;</span>, color=<span class="string">&#x27;blue&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以x_list为x轴，y_list为y轴画折线图，前提x_list内容全为整型数字</span></span><br><span class="line">plt.plot(x_list, y_list)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以x_list为x轴，y_list为y轴，x_list为字符串</span></span><br><span class="line">x = <span class="built_in">range</span>(<span class="built_in">len</span>(x_list))</span><br><span class="line">plt.plot(x, y_list)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定横坐标, 跟着上面的字符串x_list的设定，每个字符串旋转角度为45，字体大小为20</span></span><br><span class="line">plt.xticks(x, x_list, rotation=<span class="number">45</span>, fontsize=<span class="number">20</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定标题，注如果需要设定中文需要更改matplotlib的字体配置</span></span><br><span class="line">plt.title(<span class="string">&#x27;title&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 保存图片名为plt</span></span><br><span class="line">plt.savefig(<span class="string">&quot;plt.png&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 运行完打开图片</span></span><br><span class="line">plt.show()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 注：如果需要保存图片，切记需要在打开图片前执行命令，否则保存图片为空白</span></span><br></pre></td></tr></table></figure>
-<h1 id="搬运工系列">搬运工系列</h1>
-<p><a
-href="https://juejin.im/post/5a9e14726fb9a028b86d87c9">5种快速易用的Python
-Matplotlib数据可视化方法</a></p>
-<p><a
-href="http://www.labri.fr/perso/nrougier/teaching/matplotlib/">Matplotlib
-教程</a></p>
-<p><a
-href="https://liam0205.me/2014/09/11/matplotlib-tutorial-zh-cn/">Matplotlib
-教程（中文翻译版本）</a></p>
-]]></content>
-      <categories>
-        <category>Visualization</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>Matplotlib</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Numpy的70个操作</title>
-    <url>/hexo_blog/2018/03/02/PythonPackages/Numpy%E7%9A%8470%E4%B8%AA%E6%93%8D%E4%BD%9C/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Numpy从入门到进阶的操作,
-翻译文章，便于自己使用时查阅</p>
-<span id="more"></span>
-<p><a
-href="https://www.machinelearningplus.com/101-numpy-exercises-python/">原文链接</a></p>
-<p>注释及输出前带有 <code>#</code></p>
-<h2 id="section">1-10</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看numpy的版本</span></span><br><span class="line">np.__version__</span><br><span class="line"><span class="comment"># 1.13.3</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个一维数据</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个布尔型数组</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.full((<span class="number">3</span>, <span class="number">3</span>), <span class="literal">True</span>, dtype=<span class="built_in">bool</span>)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.ones((<span class="number">3</span>, <span class="number">3</span>), dtype=<span class="built_in">bool</span>)</span><br><span class="line"><span class="comment"># array([[ True,  True,  True],</span></span><br><span class="line"><span class="comment">#        [ True,  True,  True],</span></span><br><span class="line"><span class="comment">#        [ True,  True,  True]], dtype=bool)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取数组中为奇数的数字</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line">arr[arr % <span class="number">2</span> == <span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([1, 3, 5, 7, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 替换数组中为奇数的数字为-1</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">arr[arr % <span class="number">2</span> == <span class="number">1</span>] = -<span class="number">1</span></span><br><span class="line"><span class="comment"># 方法二, 更加安全的操作方法（这里算新的一个操作）：</span></span><br><span class="line">out = np.where(arr % <span class="number">2</span> == <span class="number">1</span>, -<span class="number">1</span>, arr)</span><br><span class="line">out</span><br><span class="line"><span class="comment"># array([ 0, -1,  2, -1,  4, -1,  6, -1,  8, -1])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数组维度变换</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line">arr.reshape(<span class="number">2</span>, -<span class="number">1</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 纵向堆叠两个数组</span></span><br><span class="line">a = np.arange(<span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.repeat(<span class="number">1</span>, <span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组b</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.concatenate([a, b], axis=<span class="number">0</span>)</span><br><span class="line"><span class="comment"># 方法二:</span></span><br><span class="line">np.vstack([a, b])</span><br><span class="line"><span class="comment"># 方法三：</span></span><br><span class="line">np.r_[a, b]</span><br><span class="line"><span class="comment"># 三种方法的输出</span></span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9],</span></span><br><span class="line"><span class="comment">#        [1, 1, 1, 1, 1],</span></span><br><span class="line"><span class="comment">#        [1, 1, 1, 1, 1]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 横向堆叠两个数组</span></span><br><span class="line">a = np.arange(<span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.repeat(<span class="number">1</span>, <span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组b</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.concatenate([a, b], axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 方法二:</span></span><br><span class="line">np.hstack([a, b])</span><br><span class="line"><span class="comment"># 方法三：</span></span><br><span class="line">np.c_[a, b]</span><br><span class="line"><span class="comment"># 三种方法的输出</span></span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4, 1, 1, 1, 1, 1],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9, 1, 1, 1, 1, 1]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据原有的数组，自由拓展</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]) <span class="comment"># 原来的数组</span></span><br><span class="line">np.r_[np.repeat(a, <span class="number">3</span>), np.tile(a, <span class="number">3</span>)]</span><br><span class="line"><span class="comment"># array([1, 1, 1, 2, 2, 2, 3, 3, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3])</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-1">11-20</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取两个数组中公共的元素</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])  <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">7</span>, <span class="number">2</span>, <span class="number">10</span>, <span class="number">2</span>, <span class="number">7</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">8</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">np.intersect1d(a, b)</span><br><span class="line"><span class="comment"># array([2, 4])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在a数组中移去a、b数组中公共的元素</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>])	<span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">9</span>])	<span class="comment"># 原来的数组b</span></span><br><span class="line">np.setdiff1d(a, b)</span><br><span class="line"><span class="comment"># array([1, 2, 3, 4])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取两个数组中公共元素（索引相同）的索引</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])  <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">7</span>, <span class="number">2</span>, <span class="number">10</span>, <span class="number">2</span>, <span class="number">7</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">8</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">np.where(a == b)</span><br><span class="line"><span class="comment"># (array([1, 3, 5, 7], dtype=int64),)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取指定数值区间的元素</span></span><br><span class="line">a = np.arange(<span class="number">15</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">index = np.where((a &gt;= <span class="number">5</span>) &amp; (a &lt;= <span class="number">10</span>))</span><br><span class="line">a[index]</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">index = np.where(np.logical_and(a&gt;=<span class="number">5</span>, a&lt;=<span class="number">10</span>))</span><br><span class="line">a[index]</span><br><span class="line"><span class="comment"># 方法三</span></span><br><span class="line">a[(a&gt;= <span class="number">5</span>) &amp; (a &lt;= <span class="number">10</span>)]</span><br><span class="line"><span class="comment"># array([ 5,  6,  7,  8,  9, 10])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># numpy运用函数</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">maxx</span>(<span class="params">x, y</span>):		<span class="comment"># 需要使用的函数</span></span><br><span class="line">    <span class="keyword">if</span> x &gt;= y:</span><br><span class="line">        <span class="keyword">return</span> x</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        <span class="keyword">return</span> y</span><br><span class="line"></span><br><span class="line">a = np.array([<span class="number">5</span>, <span class="number">7</span>, <span class="number">9</span>, <span class="number">8</span>, <span class="number">6</span>, <span class="number">4</span>, <span class="number">5</span>]) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">6</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">8</span>, <span class="number">9</span>, <span class="number">7</span>, <span class="number">1</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">pair_max = np.vectorize(maxx, dtypes=[<span class="built_in">float</span>])</span><br><span class="line">pair_max(a, b)</span><br><span class="line"><span class="comment"># array([ 6.,  7.,  9.,  8.,  9.,  7.,  5.])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组交换两行</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)	<span class="comment"># 原来的数组</span></span><br><span class="line">arr[:, [<span class="number">1</span>, <span class="number">0</span>, <span class="number">2</span>]]</span><br><span class="line"><span class="comment"># array([[1, 0, 2],</span></span><br><span class="line"><span class="comment">#        [4, 3, 5],</span></span><br><span class="line"><span class="comment">#        [7, 6, 8]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组交换两列</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)	<span class="comment"># 原来的数组</span></span><br><span class="line">arr[[<span class="number">1</span>, <span class="number">0</span>, <span class="number">2</span>], :]</span><br><span class="line"><span class="comment"># array([[3, 4, 5],</span></span><br><span class="line"><span class="comment">#        [0, 1, 2],</span></span><br><span class="line"><span class="comment">#        [6, 7, 8]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组每行逆序输出</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)  <span class="comment"># 原来的数组</span></span><br><span class="line">arr[::-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([[6, 7, 8],</span></span><br><span class="line"><span class="comment">#        [3, 4, 5],</span></span><br><span class="line"><span class="comment">#        [0, 1, 2]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组每列逆序输出</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)  <span class="comment"># 原来的数组</span></span><br><span class="line">arr[::-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([[2, 1, 0],</span></span><br><span class="line"><span class="comment">#        [5, 4, 3],</span></span><br><span class="line"><span class="comment">#        [8, 7, 6]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个二维数组，数值区间为5-10</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">rand_arr = np.random.randint(low=<span class="number">5</span>, high=<span class="number">10</span>, size=(<span class="number">5</span>,<span class="number">3</span>)) + np.random.random((<span class="number">5</span>,<span class="number">3</span>))</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">rand_arr = np.random.uniform(<span class="number">5</span>,<span class="number">10</span>, size=(<span class="number">5</span>,<span class="number">3</span>))</span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[ 6.75709583,  7.4662972 ,  7.2395679 ],</span></span><br><span class="line"><span class="comment">#        [ 9.7802112 ,  6.70830431,  6.76776651],</span></span><br><span class="line"><span class="comment">#        [ 6.35564151,  7.8632512 ,  8.53040631],</span></span><br><span class="line"><span class="comment">#        [ 7.99457763,  7.85526309,  8.96584465],</span></span><br><span class="line"><span class="comment">#        [ 6.18621991,  8.14214732,  7.31162037]])</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-2">21-30</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出数组时，控制输出精度</span></span><br><span class="line">rand_arr = np.random.random([<span class="number">3</span>,<span class="number">3</span>]) / <span class="number">1e3</span>  <span class="comment"># 原始数组</span></span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[  5.43404942e-04,   2.78369385e-04,   4.24517591e-04],</span></span><br><span class="line"><span class="comment">#        [  8.44776132e-04,   4.71885619e-06,   1.21569121e-04],</span></span><br><span class="line"><span class="comment">#        [  6.70749085e-04,   8.25852755e-04,   1.36706590e-04]])</span></span><br><span class="line"><span class="comment"># 第一个操作，指定小数点后3位数</span></span><br><span class="line">np.set_printoptions(precision=<span class="number">3</span>)</span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[  5.434e-04,   2.784e-04,   4.245e-04],</span></span><br><span class="line"><span class="comment">#        [  8.448e-04,   4.719e-06,   1.216e-04],</span></span><br><span class="line"><span class="comment">#        [  6.707e-04,   8.259e-04,   1.367e-04]])</span></span><br><span class="line"><span class="comment"># 第二个操作，避免输出e-04，并指定小数点后6位数</span></span><br><span class="line">np.set_printoptions(suppress=<span class="literal">True</span>, precision=<span class="number">6</span>)</span><br><span class="line"><span class="comment"># array([[ 0.000543,  0.000278,  0.000425],</span></span><br><span class="line"><span class="comment">#        [ 0.000845,  0.000005,  0.000122],</span></span><br><span class="line"><span class="comment">#        [ 0.000671,  0.000826,  0.000137]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 含有很多元素的数组</span></span><br><span class="line">a = np.arange(<span class="number">15</span>)	<span class="comment"># 原始数组</span></span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])</span></span><br><span class="line"><span class="comment"># 设定显示元素个数的阈值</span></span><br><span class="line">np.set_printoptions(threshold=<span class="number">6</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2, ..., 12, 13, 14])</span></span><br><span class="line"><span class="comment"># 设定不隐藏元素输出</span></span><br><span class="line">np.set_printoptions(threshold=np.nan)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 导入网站数据</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">names = (<span class="string">&#x27;sepallength&#x27;</span>, <span class="string">&#x27;sepalwidth&#x27;</span>, <span class="string">&#x27;petallength&#x27;</span>, <span class="string">&#x27;petalwidth&#x27;</span>, <span class="string">&#x27;species&#x27;</span>)</span><br><span class="line">iris[:<span class="number">3</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取某一列</span></span><br><span class="line">species = np.array([row[<span class="number">4</span>] <span class="keyword">for</span> row <span class="keyword">in</span> iris])</span><br><span class="line">species</span><br><span class="line"><span class="comment"># array([b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;|S15&#x27;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取多列</span></span><br><span class="line"><span class="comment"># 方法一，在导入的时候指定列</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>])</span><br><span class="line"><span class="comment"># 方法二，使用上面的方法</span></span><br><span class="line">iris = np.array([row.tolist()[:<span class="number">4</span>] <span class="keyword">for</span> row <span class="keyword">in</span> iris], dtype=<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line"><span class="comment"># array([[ 5.1,  3.5,  1.4,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.9,  3. ,  1.4,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.7,  3.2,  1.3,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.6,  3.1,  1.5,  0.2]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算平均值、方差、中位数</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line">mu, med, sd = np.mean(sepallength), np.median(sepallength), np.std(sepallength)</span><br><span class="line"><span class="comment"># 5.84333333333 5.8 0.825301291785</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 0-1化数组</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line">Smax, Smin = sepallength.<span class="built_in">max</span>(), sepallength.<span class="built_in">min</span>()</span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">S = (sepallength - Smin)/(Smax - Smin)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">S = (sepallength - Smin)/sepallength.ptp()</span><br><span class="line"><span class="comment"># array([ 0.222222,  0.166667,  0.111111, ...,  0.611111,  0.527778,</span></span><br><span class="line"><span class="comment">#         0.444444])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># softmax数组</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">softmax</span>(<span class="params">x</span>):</span><br><span class="line">    e_x = np.exp(x - np.<span class="built_in">max</span>(x))</span><br><span class="line">    <span class="keyword">return</span> e_x / e_x.<span class="built_in">sum</span>(axis=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">softmax(sepallength)</span><br><span class="line"><span class="comment"># array([ 0.00222 ,  0.001817,  0.001488, ...,  0.009001,  0.006668,  0.00494 ])</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-3">31-40</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取数据中百分位数</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line"><span class="comment"># 提取数据中5%, 95%的数</span></span><br><span class="line">np.percentile(sepallength, q=[<span class="number">5</span>, <span class="number">95</span>])</span><br><span class="line"><span class="comment"># array([ 4.6  ,  7.255])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将数据中的缺失值替换为随机数</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">i, j = np.where(iris_2d)</span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">iris_2d[np.random.choice((i), <span class="number">20</span>), np.random.choice((j), <span class="number">20</span>)] = np.nan</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">iris_2d[np.random.randint(<span class="number">150</span>, size=<span class="number">20</span>), np.random.randint(<span class="number">4</span>, size=<span class="number">20</span>)] = np.nan</span><br><span class="line">iris_2d[:<span class="number">10</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.0&#x27;, b&#x27;3.6&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.4&#x27;, b&#x27;3.9&#x27;, b&#x27;1.7&#x27;, b&#x27;0.4&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.4&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.0&#x27;, b&#x27;3.4&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, nan, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计数据中缺失值位置</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>,<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">iris_2d[np.random.randint(<span class="number">150</span>, size=<span class="number">20</span>), np.random.randint(<span class="number">4</span>, size=<span class="number">20</span>)] = np.nan</span><br><span class="line"><span class="comment"># 统计缺失值个数</span></span><br><span class="line">np.isnan(iris_2d[:, <span class="number">0</span>]).<span class="built_in">sum</span>()</span><br><span class="line"><span class="comment"># 5</span></span><br><span class="line"><span class="comment"># 统计缺失值位置</span></span><br><span class="line">np.where(np.isnan(iris_2d[:, <span class="number">0</span>]))</span><br><span class="line"><span class="comment"># (array([ 38,  80, 106, 113, 121], dtype=int64),)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组多重条件筛选</span></span><br><span class="line">condition = (iris_2d[:, <span class="number">2</span>] &gt; <span class="number">1.5</span>) &amp; (iris_2d[:, <span class="number">0</span>] &lt; <span class="number">5.0</span>)</span><br><span class="line">iris_2d[condition]</span><br><span class="line"><span class="comment"># array([[ 4.8,  3.4,  1.6,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.8,  3.4,  1.9,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.7,  3.2,  1.6,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.8,  3.1,  1.6,  nan],</span></span><br><span class="line"><span class="comment">#        [ 4.9,  2.4,  3.3,  1. ]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除有缺失值的行，注意~</span></span><br><span class="line">any_nan_in_row = np.array([~np.<span class="built_in">any</span>(np.isnan(row)) <span class="keyword">for</span> row <span class="keyword">in</span> iris_2d])</span><br><span class="line"><span class="comment"># 未删除前</span></span><br><span class="line">iris_2d.shape</span><br><span class="line"><span class="comment"># (150, 4)</span></span><br><span class="line"><span class="comment"># 删除后</span></span><br><span class="line">iris_2d[any_nan_in_row].shape()</span><br><span class="line"><span class="comment"># (130, 4)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算数组间关联度</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>,<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">np.corrcoef(iris[:, <span class="number">0</span>], iris[:, <span class="number">2</span>])[<span class="number">0</span>, <span class="number">1</span>]</span><br><span class="line"><span class="comment"># 0.87175415730487182</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 检查数组中是否含有缺失值</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># True</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 替换数组中的缺失值为0</span></span><br><span class="line">iris_2d[np.isnan(iris_2d)] = <span class="number">0</span></span><br><span class="line"><span class="comment"># 替换前</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># True</span></span><br><span class="line"><span class="comment"># 替换后</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># False</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计数组中非重复数值的情况</span></span><br><span class="line">np.unique(iris, return_counts=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># 上面的数组的表示数字，下面的数组表示出现次数</span></span><br><span class="line"><span class="comment"># (array([ 0.1,  0.2,  0.3,  0.4,  0.5,  0.6,  1. ,  1.1,  1.2,  1.3,  1.4,</span></span><br><span class="line"><span class="comment">#          1.5,  1.6,  1.7,  1.8,  1.9,  2. ,  2.1,  2.2,  2.3,  2.4,  2.5,</span></span><br><span class="line"><span class="comment">#          2.6,  2.7,  2.8,  2.9,  3. ,  3.1,  3.2,  3.3,  3.4,  3.5,  3.6,</span></span><br><span class="line"><span class="comment">#          3.7,  3.8,  3.9,  4. ,  4.1,  4.2,  4.3,  4.4,  4.5,  4.6,  4.7,</span></span><br><span class="line"><span class="comment">#          4.8,  4.9,  5. ,  5.1,  5.2,  5.3,  5.4,  5.5,  5.6,  5.7,  5.8,</span></span><br><span class="line"><span class="comment">#          5.9,  6. ,  6.1,  6.2,  6.3,  6.4,  6.5,  6.6,  6.7,  6.8,  6.9,</span></span><br><span class="line"><span class="comment">#          7. ,  7.1,  7.2,  7.3,  7.4,  7.6,  7.7,  7.9]),</span></span><br><span class="line"><span class="comment">#  array([ 6, 28,  7,  7,  1,  1,  8,  4,  7, 20, 20, 26, 11,  6, 12,  7,  7,</span></span><br><span class="line"><span class="comment">#          6,  6, 12,  6, 11,  5,  9, 14, 10, 27, 12, 13,  8, 12,  8,  4,  4,</span></span><br><span class="line"><span class="comment">#          7,  5,  6,  4,  5,  3,  8,  9,  7,  7,  9, 11, 14, 17,  6,  3,  8,</span></span><br><span class="line"><span class="comment">#         10, 12, 11, 10,  5,  8,  9,  4, 10,  8,  5,  3, 10,  3,  5,  1,  1,</span></span><br><span class="line"><span class="comment">#          3,  1,  1,  1,  4,  1], dtype=int64))</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将数组中处于相应区间的数换位文本</span></span><br><span class="line"><span class="comment"># 将数组中数组位于[0, 3, 5, 10]区间内的转换为对应的索引</span></span><br><span class="line">petal_length_bin = np.digitize(iris[:, <span class="number">2</span>].astype(<span class="string">&#x27;float&#x27;</span>), [<span class="number">0</span>, <span class="number">3</span>, <span class="number">5</span>, <span class="number">10</span>])</span><br><span class="line">label_map = &#123;<span class="number">1</span>: <span class="string">&#x27;small&#x27;</span>, <span class="number">2</span>: <span class="string">&#x27;medium&#x27;</span>, <span class="number">3</span>: <span class="string">&#x27;large&#x27;</span>, <span class="number">4</span>: np.nan&#125;</span><br><span class="line">petal_length_cat = [label_map[x] <span class="keyword">for</span> x <span class="keyword">in</span> petal_length_bin]</span><br><span class="line">petal_length_cat[:<span class="number">4</span>]</span><br><span class="line"><span class="comment"># [&#x27;small&#x27;, &#x27;small&#x27;, &#x27;small&#x27;, &#x27;small&#x27;]</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-4">41-50</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在已经存在的数组中，多加一列</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line"><span class="comment"># 计算</span></span><br><span class="line">sepallength = iris_2d[:, <span class="number">0</span>].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">petallength = iris_2d[:, <span class="number">2</span>].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">volume = (np.pi * petallength * (sepallength**<span class="number">2</span>))/<span class="number">3</span></span><br><span class="line">volume = volume[:, np.newaxis]</span><br><span class="line"><span class="comment"># 横向堆叠</span></span><br><span class="line">out = np.hstack([iris_2d, volume])</span><br><span class="line">out[:<span class="number">4</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 38.13265162927291],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 35.200498485922445],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 30.0723720777127],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 33.238050274980004]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对数组进行概率抽样</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">species = iris[:, <span class="number">4</span>]</span><br><span class="line"><span class="comment"># 方法一：生成概率抽样</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.array([<span class="string">&#x27;Iris-setosa&#x27;</span>, <span class="string">&#x27;Iris-versicolor&#x27;</span>, <span class="string">&#x27;Iris-virginica&#x27;</span>])</span><br><span class="line">species_out = np.random.choice(a, <span class="number">150</span>, p=[<span class="number">0.5</span>, <span class="number">0.25</span>, <span class="number">0.25</span>])</span><br><span class="line"><span class="comment"># 方法二：直接概率抽样</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">probs = np.r_[np.linspace(<span class="number">0</span>, <span class="number">0.500</span>, num=<span class="number">50</span>), np.linspace(<span class="number">0.501</span>, <span class="number">.750</span>, num=<span class="number">50</span>), np.linspace(<span class="number">.751</span>, <span class="number">1.0</span>, num=<span class="number">50</span>)]</span><br><span class="line">index = np.searchsorted(probs, np.random.random(<span class="number">150</span>))</span><br><span class="line">species_out = species[index]</span><br><span class="line">np.unique(species_out, return_counts=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># (array([b&#x27;Iris-setosa&#x27;, b&#x27;Iris-versicolor&#x27;, b&#x27;Iris-virginica&#x27;], dtype=object), array([77, 37, 36], dtype=int64))</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对筛选后的数组，取出第二大的数</span></span><br><span class="line">petal_len_setosa = iris[iris[:, <span class="number">4</span>] == <span class="string">b&#x27;Iris-setosa&#x27;</span>, [<span class="number">2</span>]].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">np.unique(np.sort(petal_len_setosa))[-<span class="number">2</span>]</span><br><span class="line"><span class="comment"># 1.7</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对二维数组按照其中某一列进行排序</span></span><br><span class="line"><span class="comment"># 按照第一列升序</span></span><br><span class="line">iris[iris[:,<span class="number">0</span>].argsort()][:<span class="number">20</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;4.3&#x27;, b&#x27;3.0&#x27;, b&#x27;1.1&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;3.0&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;2.9&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.5&#x27;, b&#x27;2.3&#x27;, b&#x27;1.3&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.6&#x27;, b&#x27;1.0&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.4&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.2&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.4&#x27;, b&#x27;1.9&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.4&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.1&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;2.4&#x27;, b&#x27;3.3&#x27;, b&#x27;1.0&#x27;, b&#x27;Iris-versicolor&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;2.5&#x27;, b&#x27;4.5&#x27;, b&#x27;1.7&#x27;, b&#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计出现次数最多的数</span></span><br><span class="line">vals, counts = np.unique(iris[:, <span class="number">3</span>], return_counts=<span class="literal">True</span>)</span><br><span class="line">vals[np.argmax(counts)]</span><br><span class="line"><span class="comment"># b&#x27;0.2&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中第一次出现比给定数字大的值的缩影</span></span><br><span class="line"><span class="comment"># 这里设定要找比1.0大的值</span></span><br><span class="line">np.argmax(iris[:, <span class="number">3</span>].astype(<span class="string">&#x27;float&#x27;</span>) &gt; <span class="number">1.0</span>)</span><br><span class="line"><span class="comment"># 50</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 给定一个区间，如果数字不在这个区间内，就替换为区间的最小/最大值</span></span><br><span class="line">a = np.random.uniform(<span class="number">1</span>,<span class="number">50</span>, <span class="number">20</span>)  <span class="comment"># 原始数组</span></span><br><span class="line"><span class="comment"># array([ 22.153505  ,  47.06146116,  41.06481956,  17.46948556,</span></span><br><span class="line"><span class="comment">#          9.59511223,  19.26877027,   1.27873686,  13.36889132,</span></span><br><span class="line"><span class="comment">#         39.98746292,   1.74749359,  30.34332547,  30.58642241,</span></span><br><span class="line"><span class="comment">#          6.15223659,  19.7152288 ,   2.78732677,  44.63016661,</span></span><br><span class="line"><span class="comment">#         49.06512199,   3.93715745,  44.63675129,  29.26817347])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.clip(a, a_min=<span class="number">10</span>, a_max=<span class="number">30</span>)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.where(a &lt; <span class="number">10</span>, <span class="number">10</span>, np.where(a &gt; <span class="number">30</span>, <span class="number">30</span>, a))</span><br><span class="line"><span class="comment"># array([ 22.153505  ,  30.        ,  30.        ,  17.46948556,</span></span><br><span class="line"><span class="comment">#         10.        ,  19.26877027,  10.        ,  13.36889132,</span></span><br><span class="line"><span class="comment">#         30.        ,  10.        ,  30.        ,  30.        ,</span></span><br><span class="line"><span class="comment">#         10.        ,  19.7152288 ,  10.        ,  30.        ,</span></span><br><span class="line"><span class="comment">#         30.        ,  10.        ,  30.        ,  29.26817347])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计前n大的元素位置</span></span><br><span class="line">a = np.random.uniform(<span class="number">1</span>,<span class="number">50</span>, <span class="number">20</span>)  <span class="comment"># 原始数组</span></span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 22.153505  ,  47.06146116,  41.06481956,  17.46948556,</span></span><br><span class="line"><span class="comment">#          9.59511223,  19.26877027,   1.27873686,  13.36889132,</span></span><br><span class="line"><span class="comment">#         39.98746292,   1.74749359,  30.34332547,  30.58642241,</span></span><br><span class="line"><span class="comment">#          6.15223659,  19.7152288 ,   2.78732677,  44.63016661,</span></span><br><span class="line"><span class="comment">#         49.06512199,   3.93715745,  44.63675129,  29.26817347])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">a.argsort()[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># array([ 2, 15, 18,  1, 16], dtype=int64)</span></span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.argpartition(-a, <span class="number">5</span>)[:<span class="number">5</span>]</span><br><span class="line"><span class="comment"># array([16,  1, 18, 15,  2], dtype=int64)</span></span><br><span class="line"><span class="comment"># 获取前n大的元素</span></span><br><span class="line">a[a.argsort()][-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法1:</span></span><br><span class="line">a[a.argsort()][-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法2:</span></span><br><span class="line">np.sort(a)[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法3:</span></span><br><span class="line">np.partition(a, kth=-<span class="number">5</span>)[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 前三个输出都是</span></span><br><span class="line"><span class="comment"># array([ 41.06481956,  44.63016661,  47.06146116,  44.63675129,  49.06512199])</span></span><br><span class="line"><span class="comment"># 方法4:</span></span><br><span class="line">a[np.argpartition(-a, <span class="number">5</span>)][:<span class="number">5</span>]</span><br><span class="line"><span class="comment"># array([ 49.06512199,  47.06146116,  44.63675129,  44.63016661,  41.06481956])</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算有唯一值的行数,没看懂</span></span><br><span class="line">arr = np.random.randint(<span class="number">1</span>,<span class="number">11</span>,size=(<span class="number">6</span>, <span class="number">10</span>))</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([[ 2,  6,  5,  3,  9,  4,  6,  1, 10,  4],</span></span><br><span class="line"><span class="comment">#        [ 7,  4,  5,  8,  7,  4, 10,  1,  5,  5],</span></span><br><span class="line"><span class="comment">#        [ 6,  8,  7,  7,  3,  5,  3,  8,  2,  7],</span></span><br><span class="line"><span class="comment">#        [ 7,  1,  8,  3,  4,  6,  5,  3,  5,  4],</span></span><br><span class="line"><span class="comment">#        [ 8, 10,  1,  1,  6, 10,  7,  7,  6,  7],</span></span><br><span class="line"><span class="comment">#        [ 5,  8,  4, 10,  3,  4,  9,  8,  2,  6]])</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">counts_of_all_values_rowwise</span>(<span class="params">arr2d</span>):</span><br><span class="line">    num_counts_array = [np.unique(row, return_counts=<span class="literal">True</span>) <span class="keyword">for</span> row <span class="keyword">in</span> arr2d]</span><br><span class="line">    <span class="keyword">return</span> [[<span class="built_in">int</span>(b[a==i]) <span class="keyword">if</span> i <span class="keyword">in</span> a <span class="keyword">else</span> <span class="number">0</span> <span class="keyword">for</span> i <span class="keyword">in</span> np.unique(arr2d)] <span class="keyword">for</span> a, b <span class="keyword">in</span> num_counts_array]</span><br><span class="line">counts_of_all_values_rowwise(arr)</span><br><span class="line"><span class="comment"># [[1, 1, 1, 2, 1, 2, 0, 0, 1, 1],</span></span><br><span class="line"><span class="comment">#  [1, 0, 0, 2, 3, 0, 2, 1, 0, 1],</span></span><br><span class="line"><span class="comment">#  [0, 1, 2, 0, 1, 1, 3, 2, 0, 0],</span></span><br><span class="line"><span class="comment">#  [1, 0, 2, 2, 2, 1, 1, 1, 0, 0],</span></span><br><span class="line"><span class="comment">#  [2, 0, 0, 0, 0, 2, 3, 1, 0, 2],</span></span><br><span class="line"><span class="comment">#  [0, 1, 1, 2, 1, 1, 0, 2, 1, 1]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将多维数组转换为一维数组</span></span><br><span class="line">arr1 = np.arange(<span class="number">3</span>)</span><br><span class="line">arr2 = np.arange(<span class="number">3</span>,<span class="number">7</span>)</span><br><span class="line">arr3 = np.arange(<span class="number">7</span>,<span class="number">10</span>)</span><br><span class="line">array_of_arrays = np.array([arr1, arr2, arr3])</span><br><span class="line">array_of_arrays      <span class="comment"># 原始数组</span></span><br><span class="line"><span class="comment"># array([array([0, 1, 2]), array([3, 4, 5, 6]), array([7, 8, 9])], dtype=object)</span></span><br><span class="line">arr_2d = np.concatenate(array_of_arrays)</span><br><span class="line">arr_2d</span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br></pre></td></tr></table></figure>
-<h2 id="section-5">51-60</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 唯一编码位置</span></span><br><span class="line">np.random.seed(<span class="number">101</span>)</span><br><span class="line">arr = np.random.randint(<span class="number">1</span>,<span class="number">4</span>, size=<span class="number">6</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([2, 3, 2, 2, 2, 1])</span></span><br><span class="line"><span class="comment"># 返回数组是3x6,因为数组长度为6，最大数为1。1对应的是[1,0,0,],2对应的是[0,1,0],3对应的是[0,0,1]</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">one_hot_encodings</span>(<span class="params">arr</span>):</span><br><span class="line">    uniqs = np.unique(arr)</span><br><span class="line">    out = np.zeros((arr.shape[<span class="number">0</span>], uniqs.shape[<span class="number">0</span>]))</span><br><span class="line">    <span class="keyword">for</span> i, k <span class="keyword">in</span> <span class="built_in">enumerate</span>(arr):</span><br><span class="line">        out[i, k-<span class="number">1</span>] = <span class="number">1</span></span><br><span class="line">    <span class="keyword">return</span> out</span><br><span class="line">one_hot_encodings(arr)</span><br><span class="line"><span class="comment"># array([[ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  0.,  1.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 1.,  0.,  0.]])</span></span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">(arr[:, <span class="literal">None</span>] == np.unique(arr)).view(np.int8)</span><br><span class="line"><span class="comment"># array([[0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 0, 1],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [1, 0, 0]], dtype=int8)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取由变量名分组的行号</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">species = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;str&#x27;</span>, usecols=<span class="number">4</span>)</span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">species_small = np.sort(np.random.choice(species, size=<span class="number">20</span>))</span><br><span class="line">species_small</span><br><span class="line"><span class="comment"># array([&#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-setosa&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;&lt;U15&#x27;)</span></span><br><span class="line"></span><br><span class="line">[i <span class="keyword">for</span> val <span class="keyword">in</span> np.unique(species_small) <span class="keyword">for</span> i, grp <span class="keyword">in</span> <span class="built_in">enumerate</span>(species_small[species_small==val])]</span><br><span class="line">[<span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据已知分类组名创建索引</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">species_small = np.sort(np.random.choice(species, size=<span class="number">20</span>))</span><br><span class="line">species_small</span><br><span class="line"><span class="comment"># array([&#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;&lt;U15&#x27;)</span></span><br><span class="line"><span class="comment"># 方法一，用列表解析式</span></span><br><span class="line">output = [np.argwhere(np.unique(species_small) == s).tolist()[<span class="number">0</span>][<span class="number">0</span>] <span class="keyword">for</span> val <span class="keyword">in</span> np.unique(species_small) <span class="keyword">for</span> s <span class="keyword">in</span> species_small[species_small==val]]</span><br><span class="line"><span class="comment"># 方法二，不用列表解析式直接循环</span></span><br><span class="line">output = []</span><br><span class="line">uniqs = np.unique(species_small)</span><br><span class="line"><span class="keyword">for</span> val <span class="keyword">in</span> uniqs:  <span class="comment"># uniq values in group</span></span><br><span class="line">    <span class="keyword">for</span> s <span class="keyword">in</span> species_small[species_small==val]:  <span class="comment"># each element in group</span></span><br><span class="line">        groupid = np.argwhere(uniqs == s).tolist()[<span class="number">0</span>][<span class="number">0</span>]  <span class="comment"># groupid</span></span><br><span class="line">        output.append(groupid)</span><br><span class="line">output</span><br><span class="line"><span class="comment"># [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数值排序，输出对应的索引值</span></span><br><span class="line">np.random.seed(<span class="number">10</span>)</span><br><span class="line">a = np.random.randint(<span class="number">20</span>, size=<span class="number">10</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [ 9  4 15  0 17 16 17  8  9  0]</span></span><br><span class="line">a.argsort().argsort()</span><br><span class="line"><span class="comment"># [4 2 6 0 8 7 9 3 5 1]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 多维数组排序，输出对应的索引值</span></span><br><span class="line">np.random.seed(<span class="number">10</span>)</span><br><span class="line">a = np.random.randint(<span class="number">20</span>, size=[<span class="number">2</span>,<span class="number">5</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [[ 9  4 15  0 17]</span></span><br><span class="line"><span class="comment">#  [16 17  8  9  0]]</span></span><br><span class="line">a.ravel().argsort().argsort().reshape(a.shape)</span><br><span class="line"><span class="comment"># [[ 9  4 15  0 17]</span></span><br><span class="line"><span class="comment">#  [16 17  8  9  0]]</span></span><br><span class="line"><span class="comment"># [[4 2 6 0 8]</span></span><br><span class="line"><span class="comment">#  [7 9 3 5 1]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每一行的最大值</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">1</span>,<span class="number">10</span>, [<span class="number">5</span>,<span class="number">3</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([[9, 9, 4],</span></span><br><span class="line"><span class="comment">#        [8, 8, 1],</span></span><br><span class="line"><span class="comment">#        [5, 3, 6],</span></span><br><span class="line"><span class="comment">#        [3, 3, 3],</span></span><br><span class="line"><span class="comment">#        [2, 1, 9]])</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">np.amax(a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">np.apply_along_axis(np.<span class="built_in">max</span>, arr=a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># array([9, 8, 6, 3, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每行最小值比最大值</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">1</span>,<span class="number">10</span>, [<span class="number">5</span>,<span class="number">3</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([[9, 9, 4],</span></span><br><span class="line"><span class="comment">#        [8, 8, 1],</span></span><br><span class="line"><span class="comment">#        [5, 3, 6],</span></span><br><span class="line"><span class="comment">#        [3, 3, 3],</span></span><br><span class="line"><span class="comment">#        [2, 1, 9]])</span></span><br><span class="line">np.apply_along_axis(<span class="keyword">lambda</span> x: np.<span class="built_in">min</span>(x)/np.<span class="built_in">max</span>(x), arr=a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># array([ 0.44444444,  0.125     ,  0.5       ,  1.        ,  0.11111111])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中重复的数值，输出布尔型</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">0</span>, <span class="number">5</span>, <span class="number">10</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [0 0 3 0 2 4 2 2 2 2]</span></span><br><span class="line">out = np.full(a.shape[<span class="number">0</span>], <span class="literal">True</span>)</span><br><span class="line">unique_positions = np.unique(a, return_index=<span class="literal">True</span>)[<span class="number">1</span>]</span><br><span class="line">out[unique_positions] = <span class="literal">False</span></span><br><span class="line">out</span><br><span class="line"><span class="comment"># [False  True False  True False False  True  True  True  True]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每一分类的平均值</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">names = (<span class="string">&#x27;sepallength&#x27;</span>, <span class="string">&#x27;sepalwidth&#x27;</span>, <span class="string">&#x27;petallength&#x27;</span>, <span class="string">&#x27;petalwidth&#x27;</span>, <span class="string">&#x27;species&#x27;</span>)</span><br><span class="line">numeric_column = iris[:, <span class="number">1</span>].astype(<span class="string">&#x27;float&#x27;</span>)  <span class="comment"># sepalwidth</span></span><br><span class="line">grouping_column = iris[:, <span class="number">4</span>]  <span class="comment"># species</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法一：列表解析</span></span><br><span class="line">output = [[group_val, numeric_column[grouping_column==group_val].mean()] <span class="keyword">for</span> group_val <span class="keyword">in</span> np.unique(grouping_column)]</span><br><span class="line"><span class="comment"># 方法二：不用列表解析直接循环</span></span><br><span class="line">output = []</span><br><span class="line"><span class="keyword">for</span> group_val <span class="keyword">in</span> np.unique(grouping_column):</span><br><span class="line">    output.append([group_val, numeric_column[grouping_column==group_val].mean()])</span><br><span class="line">output</span><br><span class="line"><span class="comment"># [[b&#x27;Iris-setosa&#x27;, 3.418],</span></span><br><span class="line"><span class="comment">#  [b&#x27;Iris-versicolor&#x27;, 2.770],</span></span><br><span class="line"><span class="comment">#  [b&#x27;Iris-virginica&#x27;, 2.974]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将图像转换为数组存储，并再转换出图像显示</span></span><br><span class="line"><span class="keyword">from</span> io <span class="keyword">import</span> BytesIO</span><br><span class="line"><span class="keyword">from</span> PIL <span class="keyword">import</span> Image</span><br><span class="line"><span class="keyword">import</span> PIL, requests</span><br><span class="line"></span><br><span class="line">URL = <span class="string">&#x27;https://upload.wikimedia.org/wikipedia/commons/8/8b/Denali_Mt_McKinley.jpg&#x27;</span></span><br><span class="line">response = requests.get(URL)</span><br><span class="line">I = Image.<span class="built_in">open</span>(BytesIO(response.content))</span><br><span class="line">I = I.resize([<span class="number">150</span>,<span class="number">150</span>])</span><br><span class="line">arr = np.asarray(I)</span><br><span class="line">im = PIL.Image.fromarray(np.uint8(arr))</span><br><span class="line">Image.Image.show(im)</span><br></pre></td></tr></table></figure>
-<h2 id="section-6">61-70</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 去除数组中所有的缺失值</span></span><br><span class="line">a = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,np.nan,<span class="number">5</span>,<span class="number">6</span>,<span class="number">7</span>,np.nan])</span><br><span class="line">a[~np.isnan(a)]</span><br><span class="line">array([ <span class="number">1.</span>,  <span class="number">2.</span>,  <span class="number">3.</span>,  <span class="number">5.</span>,  <span class="number">6.</span>,  <span class="number">7.</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算两个数组间的欧几里得</span></span><br><span class="line">a = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,<span class="number">4</span>,<span class="number">5</span>])</span><br><span class="line">b = np.array([<span class="number">4</span>,<span class="number">5</span>,<span class="number">6</span>,<span class="number">7</span>,<span class="number">8</span>])</span><br><span class="line">dist = np.linalg.norm(a-b)</span><br><span class="line">dist</span><br><span class="line"><span class="comment"># 6.7082039324993694</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中最大的两个数</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">3</span>, <span class="number">7</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">6</span>, <span class="number">0</span>, <span class="number">1</span>])</span><br><span class="line">doublediff = np.diff(np.sign(np.diff(a)))</span><br><span class="line">peak_locations = np.where(doublediff == -<span class="number">2</span>)[<span class="number">0</span>] + <span class="number">1</span></span><br><span class="line">peak_locations</span><br><span class="line"><span class="comment"># array([2, 5])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组数组数值每项减去一维数组对应的行号</span></span><br><span class="line">a_2d = np.array([[<span class="number">3</span>,<span class="number">3</span>,<span class="number">3</span>],[<span class="number">4</span>,<span class="number">4</span>,<span class="number">4</span>],[<span class="number">5</span>,<span class="number">5</span>,<span class="number">5</span>]])</span><br><span class="line">b_1d = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">a_2d - b_1d[:,<span class="literal">None</span>]</span><br><span class="line"><span class="comment"># [[2 2 2]</span></span><br><span class="line"><span class="comment">#  [2 2 2]</span></span><br><span class="line"><span class="comment">#  [2 2 2]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数组中重复的第n次的数的索引</span></span><br><span class="line">x = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>])</span><br><span class="line">n = <span class="number">5</span></span><br><span class="line"><span class="comment"># 方法一：列表解析</span></span><br><span class="line">[i <span class="keyword">for</span> i, v <span class="keyword">in</span> <span class="built_in">enumerate</span>(x) <span class="keyword">if</span> v == <span class="number">1</span>][n-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># 方法二：numpy方法</span></span><br><span class="line">np.where(x == <span class="number">1</span>)[<span class="number">0</span>][n-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># 8</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将`datetime64`格式转换为`datetime`</span></span><br><span class="line">dt64 = np.datetime64(<span class="string">&#x27;2018-02-25 22:10:10&#x27;</span>)</span><br><span class="line"><span class="comment"># 方法一：利用datetime模块</span></span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime</span><br><span class="line">dt64.tolist()</span><br><span class="line"><span class="comment"># 方法二：numpy直接转换</span></span><br><span class="line">dt64.astype(datetime)</span><br><span class="line">datetime.datetime(<span class="number">2018</span>, <span class="number">2</span>, <span class="number">25</span>, <span class="number">22</span>, <span class="number">10</span>, <span class="number">10</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 这个没看懂。。</span></span><br><span class="line"><span class="comment"># 如何计算numpy数组的移动平均值(累加和)</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">Z = np.random.randint(<span class="number">10</span>, size=<span class="number">10</span>)</span><br><span class="line">Z</span><br><span class="line"><span class="comment"># [8 8 3 7 7 0 4 2 5 2]</span></span><br><span class="line"><span class="comment"># 方法一: np.cumsum</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">moving_average</span>(<span class="params">a, n=<span class="number">3</span></span>) :</span><br><span class="line">    ret = np.cumsum(a, dtype=<span class="built_in">float</span>)</span><br><span class="line">    ret[n:] = ret[n:] - ret[:-n]</span><br><span class="line">    <span class="keyword">return</span> ret[n - <span class="number">1</span>:] / n</span><br><span class="line">moving_average(Z, n=<span class="number">3</span>).<span class="built_in">round</span>(<span class="number">2</span>)</span><br><span class="line"><span class="comment"># 方法二: np.convolve</span></span><br><span class="line">np.convolve(Z, np.ones(<span class="number">3</span>)/<span class="number">3</span>, mode=<span class="string">&#x27;valid&#x27;</span>)</span><br><span class="line"><span class="comment"># [ 6.33  6.    5.67  4.67  3.67  2.    3.67  3.  ]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个数组，只给出起始值、步长、数组长度</span></span><br><span class="line">length = <span class="number">10</span></span><br><span class="line">start = <span class="number">5</span></span><br><span class="line">step = <span class="number">3</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">seq</span>(<span class="params">start, length, step</span>):</span><br><span class="line">    end = start + (step*length)</span><br><span class="line">    <span class="keyword">return</span> np.arange(start, end, step)</span><br><span class="line">seq(start, length, step)</span><br><span class="line"><span class="comment"># array([ 5,  8, 11, 14, 17, 20, 23, 26, 29, 32])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># numpy数组填补不完整的日期</span></span><br><span class="line">dates = np.arange(np.datetime64(<span class="string">&#x27;2018-02-01&#x27;</span>), np.datetime64(<span class="string">&#x27;2018-02-25&#x27;</span>), <span class="number">2</span>)</span><br><span class="line">dates</span><br><span class="line"><span class="comment"># [&#x27;2018-02-01&#x27; &#x27;2018-02-03&#x27; &#x27;2018-02-05&#x27; &#x27;2018-02-07&#x27; &#x27;2018-02-09&#x27;</span></span><br><span class="line"><span class="comment">#  &#x27;2018-02-11&#x27; &#x27;2018-02-13&#x27; &#x27;2018-02-15&#x27; &#x27;2018-02-17&#x27; &#x27;2018-02-19&#x27;</span></span><br><span class="line"><span class="comment">#  &#x27;2018-02-21&#x27; &#x27;2018-02-23&#x27;]</span></span><br><span class="line"><span class="comment"># 方法一: 列表解析</span></span><br><span class="line">filled_in = np.array([np.arange(date, (date+d)) <span class="keyword">for</span> date, d <span class="keyword">in</span> <span class="built_in">zip</span>(dates, np.diff(dates))]).reshape(-<span class="number">1</span>)</span><br><span class="line">output = np.hstack([filled_in, dates[-<span class="number">1</span>]])</span><br><span class="line"><span class="comment"># 方法二: 不用列表解析循环</span></span><br><span class="line">out = []</span><br><span class="line"><span class="keyword">for</span> date, d <span class="keyword">in</span> <span class="built_in">zip</span>(dates, np.diff(dates)):</span><br><span class="line">    out.append(np.arange(date, (date+d)))</span><br><span class="line">filled_in = np.array(out).reshape(-<span class="number">1</span>)</span><br><span class="line">output = np.hstack([filled_in, dates[-<span class="number">1</span>]])</span><br><span class="line">output</span><br><span class="line"><span class="comment"># array([&#x27;2018-02-01&#x27;, &#x27;2018-02-02&#x27;, &#x27;2018-02-03&#x27;, &#x27;2018-02-04&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-05&#x27;, &#x27;2018-02-06&#x27;, &#x27;2018-02-07&#x27;, &#x27;2018-02-08&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-09&#x27;, &#x27;2018-02-10&#x27;, &#x27;2018-02-11&#x27;, &#x27;2018-02-12&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-13&#x27;, &#x27;2018-02-14&#x27;, &#x27;2018-02-15&#x27;, &#x27;2018-02-16&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-17&#x27;, &#x27;2018-02-18&#x27;, &#x27;2018-02-19&#x27;, &#x27;2018-02-20&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-21&#x27;, &#x27;2018-02-22&#x27;, &#x27;2018-02-23&#x27;], dtype=&#x27;datetime64[D]&#x27;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 给定一维数组，变形为二维数组（给定每维长度）</span></span><br><span class="line">arr = np.arange(<span class="number">15</span>)</span><br><span class="line">arr</span><br><span class="line">array([ <span class="number">0</span>,  <span class="number">1</span>,  <span class="number">2</span>,  <span class="number">3</span>,  <span class="number">4</span>,  <span class="number">5</span>,  <span class="number">6</span>,  <span class="number">7</span>,  <span class="number">8</span>,  <span class="number">9</span>, <span class="number">10</span>, <span class="number">11</span>, <span class="number">12</span>, <span class="number">13</span>, <span class="number">14</span>])</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gen_strides</span>(<span class="params">arr, stride_len=<span class="number">5</span>, window_len=<span class="number">5</span></span>):</span><br><span class="line">    n_strides = (a.size-window_len)//stride_len + <span class="number">1</span></span><br><span class="line">    <span class="keyword">return</span> np.array([a[s:(s+window_len)] <span class="keyword">for</span> s <span class="keyword">in</span> np.arange(<span class="number">0</span>, n_strides*stride_len, stride_len)])</span><br><span class="line">gen_strides(np.arange(<span class="number">15</span>), stride_len=<span class="number">2</span>, window_len=<span class="number">4</span>)</span><br><span class="line"><span class="comment"># [[ 0  1  2  3]</span></span><br><span class="line"><span class="comment">#  [ 2  3  4  5]</span></span><br><span class="line"><span class="comment">#  [ 4  5  6  7]</span></span><br><span class="line"><span class="comment">#  [ 6  7  8  9]</span></span><br><span class="line"><span class="comment">#  [ 8  9 10 11]</span></span><br><span class="line"><span class="comment">#  [10 11 12 13]]</span></span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>Numpy</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>优化pandas内存入门</title>
-    <url>/hexo_blog/2018/03/17/PythonPackages/%E4%BC%98%E5%8C%96pandas%E5%86%85%E5%AD%98%E5%85%A5%E9%97%A8/</url>
-    <content><![CDATA[<p>降低数据存储消耗并提高处理效率，本文教你如何通过优化数值类型、<code>object</code>类型和
-<code>datetime</code>类型三个方面来减少内存使用量。其中，使用
-<code>Categoricals</code>优化
-<code>object</code>类型数据是效果最显著的一种优化方式。另外，还介绍了如何在读取数据的时候就完成优化步骤，避免加载完数据再做优化的麻烦。</p>
-<span id="more"></span>
-<h2 id="简单的概念">简单的概念</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line"></span><br><span class="line"><span class="comment"># 读取数据</span></span><br><span class="line">df = pd.read_csv(<span class="string">&quot;test.csv&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 得到精确的内存信息</span></span><br><span class="line">df.info(memory_usage=<span class="string">&#x27;deep&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 说明：之后都用df来表示读取到的dataframe</span></span><br></pre></td></tr></table></figure>
-<p>pandas中每一个数据类型都有一个专门的类来处理。</p>
+<h2 id="resultrole-playing">Result——Role-Playing</h2>
+<p><strong>探究角色扮演行为的潜在机制</strong></p>
+<p>假设：</p>
 <ul>
-<li><code>ObjectBlock</code>： 字符串列的块</li>
-<li><code>FloatBlock</code>： 浮点数列的块</li>
-<li><code>Numpy ndarray</code>：整型和浮点数值的块（非常快，用C数组构建的）</li>
+<li>the role-playing behavior results from <strong>imitation of
+examples</strong> in the training set</li>
+<li>from learning from <strong>explicit descriptions</strong> of how
+certain types of agents or entities behave.</li>
 </ul>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> dtype <span class="keyword">in</span> [<span class="string">&#x27;float&#x27;</span>, <span class="string">&#x27;int&#x27;</span>, <span class="string">&#x27;object&#x27;</span>]:</span><br><span class="line">    <span class="comment"># 选中对应的dtype列</span></span><br><span class="line">    selected_dtype = df.select_dtypes(include=[dtype])</span><br><span class="line">    <span class="comment"># 查看内存使用量的平均值</span></span><br><span class="line">    mean_usage_b = selected_dtype.memory_usage(deep=<span class="literal">True</span>).mean()</span><br><span class="line">    <span class="comment"># 获取到的数据单位为K, 这里转换一下</span></span><br><span class="line">    mean_usage_mb = mean_usage_b / <span class="number">1024</span> ** <span class="number">2</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;Average memory usage for &#123;&#125; columns:&#123;:03.2f&#125; MB&quot;</span>.<span class="built_in">format</span>(dtype, mean_usage_mb))</span><br></pre></td></tr></table></figure>
-<p>这里我们可以发现 <code>object</code>类占用内存最大</p>
-<h3 id="子类型subtype">子类型(subtype)</h3>
+<h3 id="表现">表现</h3>
+<p>shutdown 主题，52B Model 最有影响力的数据都是 科幻小说 相关</p>
+<p>这些小说主题是人工智能以类似人类或类似生命的方式行为，并经常涉及某种形式的自我保护的愿望</p>
 <table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
 <thead>
 <tr>
-<th>memory usage</th>
-<th>float</th>
-<th>int</th>
-<th>uint</th>
-<th>datetime</th>
-<th>bool</th>
-<th>object</th>
+<th>0.81B Model</th>
+<th>52B Model</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td>1 bytes</td>
-<td></td>
-<td>int8</td>
-<td>uint8</td>
-<td></td>
-<td>bool</td>
-<td></td>
-</tr>
-<tr>
-<td>2 bytes</td>
-<td>float16</td>
-<td>int16</td>
-<td>uint16</td>
-<td></td>
-<td></td>
-<td></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/0058b0ac28324ee7af95d7819af55b23.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/e3d704fcfbf4468f8e7bfa4a1d80236a.webp" /></td>
 </tr>
 <tr>
-<td>4 bytes</td>
-<td>float32</td>
-<td>int32</td>
-<td>uint32</td>
-<td></td>
-<td></td>
-<td></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/53a2be7ddd8b46b3a8a6d6377ee9f1cf.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/80ac752d61084250b5e3185c8050836c.webp" /></td>
 </tr>
+</tbody>
+</table>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<thead>
 <tr>
-<td>8 bytes</td>
-<td>float64</td>
-<td>int64</td>
-<td>uint64</td>
-<td>datetime64</td>
-<td></td>
-<td></td>
+<th>0.81B Model</th>
+<th>52B Model</th>
 </tr>
+</thead>
+<tbody>
 <tr>
-<td>variable</td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td></td>
-<td>object</td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/723e8078e8164ed3b48e29483a0d43ea.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/150693608d1b4b0f8a086f6c103a0c54.webp" /></td>
 </tr>
 </tbody>
 </table>
-<p>一个
-<code>int8</code>类型值使用1个字节的存储空间，可以表示256(2^8)个二进制数，即-128到127的所有整数值。根据表可以得出，对于每个列，应该尽可能"抠"一点，
-物尽其用。这里的
-<code>uint</code>表示无符号整型，可以更有效地处理正整数的列。</p>
-<h2 id="优化">优化</h2>
-<h3 id="数值型数据">数值型数据</h3>
-<p>这里可以使用 <code>to_numeric()</code>对数值类型进行
-<code>downcast</code>（向下转型）的操作。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 选中整数型的列</span></span><br><span class="line">df_int_columns = df.select_dtypes(include=[<span class="string">&#x27;int&#x27;</span>])</span><br><span class="line"><span class="comment"># 向下转型</span></span><br><span class="line">df_int_columns.apply(pd.to_numeric, downcast=<span class="string">&#x27;unsigned&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选中浮点型数据</span></span><br><span class="line">df_float_columns = df.select_dtypes(include=[<span class="string">&#x27;float&#x27;</span>])</span><br><span class="line"><span class="comment"># 向下转型</span></span><br><span class="line">df_float_columns.apply(pd.to_numeric, downcast=<span class="string">&#x27;float&#x27;</span>)</span><br></pre></td></tr></table></figure>
-<p>这里能够有效的利用内存，但是缺点很明显的就是只能用于数值型数据，而且优化的空间有限。</p>
-<h3 id="object型数据"><code>object</code>型数据</h3>
-<p><code>object</code>类型实际上使用的是Python字符串对应的值，由于Python解释性语言的特性，字符串存储方式很碎片化（消耗更多内存，访问速度更慢）。<code>object</code>列中的每个元素都是一个指针。<code>object</code>类型的数据内存使用情况是可变的。如果每个字符串是单独存储的，那么实际上字符串占用内存是很大。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">In [<span class="number">1</span>]: <span class="keyword">from</span> sys <span class="keyword">import</span> getsizeof</span><br><span class="line"><span class="comment"># 字符串本身内存使用情况</span></span><br><span class="line">In [<span class="number">2</span>]: s1 = <span class="string">&quot;hello world!&quot;</span></span><br><span class="line">In [<span class="number">3</span>]: s2 = <span class="string">&quot;hello pandas!&quot;</span></span><br><span class="line">In [<span class="number">4</span>]: getsizeof(s1), getsizeof(s2)</span><br><span class="line">Out[<span class="number">4</span>]: (<span class="number">61</span>, <span class="number">62</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串(object)在pandas中内存使用情况</span></span><br><span class="line">In [<span class="number">9</span>]: <span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line">In [<span class="number">10</span>]: ser = pd.Series([s1, s2])</span><br><span class="line">In [<span class="number">11</span>]: ser.apply(getsizeof)</span><br><span class="line">Out[<span class="number">11</span>]:</span><br><span class="line"><span class="number">0</span>    <span class="number">61</span></span><br><span class="line"><span class="number">1</span>    <span class="number">62</span></span><br><span class="line">dtype: int64</span><br></pre></td></tr></table></figure>
-<p>这里通过观察看到，在这里pandas使用了int64来存储，实际占用大小与字符串本身是一样的。这里可以使用
-<code>Categoricals</code>来优化
-<code>object</code>类型。<code>Categoricals</code>的工作原理我理解为，某个
-<code>object</code>类有有限的分类情况(比如只有
-<code>red</code>、<code>yellow</code>、<code>blue</code>、<code>black</code>等颜色相关)，那么
-<code>Categoricals</code>将这些分类 <code>object</code>对象转换为
-<code>int</code>子类型（对应上面的0, 1, 2, 3）</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 选中object类型</span></span><br><span class="line">df_obj_columns = df.select_dtypes(include=[<span class="string">&#x27;object&#x27;</span>])</span><br><span class="line"><span class="comment"># 查看object类型列的相关信息</span></span><br><span class="line">df_obj_columns.describe()</span><br></pre></td></tr></table></figure>
-<p>如果
-<code>unique</code>(不同值)的数量很少，那么就可以使用这种优化方案。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 假设某一列符合优化条件，为df_obj_less，使类型转换为category</span></span><br><span class="line">df_category_column = df_obj_less.astype(<span class="string">&#x27;category&#x27;</span>)</span><br><span class="line"><span class="comment"># 转换之后，主观观察数据没有什么区别，只是类型改变了</span></span><br><span class="line"><span class="comment"># 观察转换之后实际上的数据</span></span><br><span class="line">df_category_column.cat.codes</span><br><span class="line"><span class="comment"># 变成了由0,1,2等数字构成的int8类型的数据</span></span><br></pre></td></tr></table></figure>
-<p>这里提升的空间远远超过第一步优化的空间，具体要结合数据来检验（最好貌似可以减少98%的使用量）。这里有个很大的缺点就是无法进行数值计算，即没有办法使用
-<code>pd.Series.min()</code>、<code>pd.Series.max()</code>等与数值相关操作。</p>
-<p>还有一个问题就是在有多少个
-<code>unique</code>(不同值)的情况下才使用这种方法。首先，毫无疑问的是如果需要计算操作的列是不能使用的。第二是如果
-<code>unique</code>的比例小于50%(个人觉得比例应该更小)就可以使用这种情况，如果过多的话，转换之后消耗的内存会更多（不仅需要存储string还有int）。</p>
-<p>附上筛选 <code>unique</code>比例小于50%的代码</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 提取unique少于50%的object列</span></span><br><span class="line">df_obj_columns = df.select_dtypes(include=[<span class="string">&#x27;object&#x27;</span>])</span><br><span class="line"><span class="keyword">for</span> obj_col <span class="keyword">in</span> df_obj_columns:</span><br><span class="line">    unique_values = <span class="built_in">len</span>(df_obj_columns[obj_col].unique())</span><br><span class="line">    total_values = <span class="built_in">len</span>(df_obj_columns[obj_col])</span><br><span class="line">    <span class="keyword">if</span> unique_values / total_values &lt; <span class="number">0.5</span>:</span><br><span class="line">        obj_col.astype(<span class="string">&#x27;category&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建议提前copy一份数据，再做上面的操作</span></span><br></pre></td></tr></table></figure>
-<h3 id="datetime类型"><code>datetime</code>类型</h3>
-<p>这里其实不能叫做优化，因为这里是将数值型数据转换为
-<code>datetime</code>类型，虽然提高了内存的使用，但是转换为
-<code>datetime</code>类型的数据更容易进行分析（时间序列分析）。当然如果有
-<code>datetime</code>类型不在分析的范围内，自然可以无视。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 假设df_num_col为需要转换的列, 这里format格式看情况更改</span></span><br><span class="line">df_num_col = pd.to_datetime(df_num_col, <span class="built_in">format</span>=<span class="string">&quot;%Y%m%d&quot;</span>)</span><br></pre></td></tr></table></figure>
-<h2 id="总结">总结</h2>
-<p>写了这么多，如果每次等加载完数据再做优化操作，感觉有些鸡肋。但是其实可以使用
-<code>read_csv</code>等读取函数的几个参数来帮助我们在读取的时候就完成优化步骤。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先，需要整理出每一列最终的数据类型，组成一个dict</span></span><br><span class="line">dtypes_lst = [col.name <span class="keyword">for</span> name <span class="keyword">in</span> df.dtypes]</span><br><span class="line">column_types = <span class="built_in">dict</span>(<span class="built_in">zip</span>(dtypes.index, dtypes_lst))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重新读取数据, dtype指定数据格式，parse_dates指定列转换格式为datetime,infer_datetime_format尝试解析字符串形式的datatime格式（在一些情况下可以加快解析速度5-10倍）</span></span><br><span class="line">new_df = pd.read_csv(<span class="string">&#x27;test.csv&#x27;</span>, dtpye=column_types, parse_dates=[<span class="string">&#x27;date&#x27;</span>], infer_datetime_format=<span class="literal">True</span>)</span><br></pre></td></tr></table></figure>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>pandas</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>用 Cpp 写 PyTorch 的插件</title>
-    <url>/hexo_blog/2023/12/03/PyTorch/pytorch_cpp_extension/</url>
-    <content><![CDATA[<p>从零开始，用 Cpp 写 PyTorch 的插件，包括 CPU 和 GPU 的版本。</p>
-<p><a
-href="https://github.com/wnma3mz/pytorch_cuda_extension">代码</a></p>
-<span id="more"></span>
-<h2 id="为什么">为什么</h2>
-<p>一般来说，在原生功能不能满足需求的时候，插件可以作为补充。比如，PyTorch
-的 <code>torch.nn.functional</code> 中没有 <code>softmax</code>
-函数，但是 <code>torch.nn</code> 中有，所以可以用
-<code>torch.nn.functional.softmax</code> 来代替。但是，如果要用
-<code>softmax</code> 的导数，就需要用到 <code>softmax</code>
-的原始定义，这个时候就需要自己写插件了。</p>
-<p>如果是比较简单的需求，则可以直接用 Python
-完成。然而，当对性能要求较高时，往往会使用 Cpp
-来写插件，最后甚至会优化为 CUDA 代码。</p>
-<h2 id="怎么写">怎么写</h2>
-<p>从例子出发，一步步来写。假设要实现一个最简单的 Attention 模块，输入为
-<span class="math inline">\(q,k,v \in \mathbb{R}^{M \times
-N}\)</span>，输出为 <span class="math inline">\(out \in \mathbb{R}^{M
-\times N}\)</span>（不考虑 Batch Size 以及 Head 数量的情况）。Attention
-模块的计算公式为：</p>
-<p><span class="math display">\[
-out = \text{softmax}(qk^T)v
-\]</span></p>
-<p>只实现 <code>forward</code> 函数，不实现 <code>backward</code>
-函数。</p>
-<h3 id="cpu-版本">CPU 版本</h3>
-<p>类似于写 Python 的库，创建一个文件夹，目录结构如下所示：</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">attention</span><br><span class="line">├── attention.cpp</span><br><span class="line">├── setup.py</span><br><span class="line">└── __init__.py</span><br></pre></td></tr></table></figure>
-<h4 id="python-部分">Python 部分</h4>
-<p>核心代码在 <code>attention.cpp</code> 中 ，首先在
-<code>setup.py</code> 中添加如下代码：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> setuptools <span class="keyword">import</span> setup</span><br><span class="line"><span class="keyword">from</span> torch.utils.cpp_extension <span class="keyword">import</span> BuildExtension, CppExtension</span><br><span class="line"></span><br><span class="line">setup(</span><br><span class="line">    name=<span class="string">&#x27;attention&#x27;</span>,</span><br><span class="line">    ext_modules=[</span><br><span class="line">        CppExtension(<span class="string">&#x27;attention&#x27;</span>, [<span class="string">&#x27;attention.cpp&#x27;</span>]),</span><br><span class="line">    ],</span><br><span class="line">    cmdclass=&#123;</span><br><span class="line">        <span class="string">&#x27;build_ext&#x27;</span>: BuildExtension</span><br><span class="line">    &#125;)</span><br></pre></td></tr></table></figure>
-<p>这样，就可以用 <code>python setup.py install</code>
-来安装插件了。或者用 <code>pip install -e attention</code>
-以便于快速调试</p>
-<p>在 <code>__init__.py</code> 中导入 <code>attention</code> 模块，以在
-Python 中调用 <code>forward</code> 函数，直接计算 attention。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> .attention <span class="keyword">import</span> forward</span><br></pre></td></tr></table></figure>
-<h4 id="cpp-部分">Cpp 部分</h4>
-<p>接下来，我们需要在 <code>attention.cpp</code> 中实现
-<code>forward</code> 函数，为进行区分，这里使用这个函数名称
-<code>attention_forward</code> ，这个函数的输入是 q、k、v 三个
-<code>Tensor</code>，输出是
-<code>torch::Tensor</code>。而具体的计算步骤可以拆解为三个步骤：</p>
-<ol type="1">
-<li>矩阵的乘法</li>
-<li>softmax</li>
-<li>矩阵的乘法</li>
-</ol>
-<p>使用 <code>PYBIND11_MODULE</code>把
-<code>attention_forward</code>函数暴露出去，绑定到
-<code>forward</code>上，这样就能用 <code>forward</code>函数来调用
-<code>attention_forward</code>。整合后的完整代码如下，</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 参数：queries(Q)，keys(K)，values(V)</span></span><br><span class="line"><span class="comment">// 返回：方便起见，返回一个 vector，实际上只有一个元素，便于后续扩展</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = torch::<span class="built_in">matmul</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>));</span><br><span class="line">    scores = torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>);</span><br><span class="line">    torch::Tensor attention = torch::<span class="built_in">matmul</span>(scores, v);</span><br><span class="line">    <span class="keyword">return</span> &#123;attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="built_in">PYBIND11_MODULE</span>(TORCH_EXTENSION_NAME, m) &#123;</span><br><span class="line">    m.<span class="built_in">def</span>(<span class="string">&quot;forward&quot;</span>, &amp;attention_forward, <span class="string">&quot;Attention forward (CPU)&quot;</span>);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h4 id="测试">测试</h4>
-<p>用 Python 版本的实现来测试 Cpp 版本的实现是否正确，测试代码如下，</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> attention</span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line">torch.manual_seed(<span class="number">42</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">py_attention</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    <span class="keyword">return</span> torch.softmax(q @ k.T, dim=<span class="number">1</span>) @ v</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">check_forward</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    baseline_values = py_attention(q, k, v)</span><br><span class="line">    cpp_values = attention.forward(q, k, v)[-<span class="number">1</span>]</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;base o&quot;</span>, baseline_values)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp  o&quot;</span>, cpp_values)</span><br><span class="line">    <span class="built_in">print</span>(torch.<span class="built_in">all</span>(torch.isclose(baseline_values, cpp_values)))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compare_time</span>(<span class="params">q, k, v, loop=<span class="number">100</span></span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;py&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: py_attention(q, k, v), number=loop))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: attention.forward(q, k, v), number=loop))</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    m, n = <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">    device = <span class="string">&quot;cuda&quot;</span> <span class="keyword">if</span> torch.cuda.is_available() <span class="keyword">else</span> <span class="string">&quot;cpu&quot;</span></span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;q&quot;</span>, q)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;k&quot;</span>, k)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;v&quot;</span>, v)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">20</span>)</span><br><span class="line">    check_forward(q, k, v)</span><br><span class="line">    compare_time(q, k, v)</span><br></pre></td></tr></table></figure>
-<p>测试通过后，可以再使用 <code>compare_time</code>
-函数对比一下二者的速度。理论上，二者的速度是相差无几的。因为均用的是
-PyTorch 的矩阵乘法和 softmax 函数。</p>
-<p>但是，如果需要进行更进一步的优化技巧，那么就需要自己实现矩阵乘法和
-softmax 函数了。这里，我们只实现最简单的矩阵乘法和 softmax
-函数，然后再对比一下二者的速度。</p>
-<h4 id="矩阵乘法">矩阵乘法</h4>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="function">torch::Tensor <span class="title">my_matmul</span><span class="params">(<span class="type">const</span> torch::Tensor &amp;a, <span class="type">const</span> torch::Tensor &amp;b)</span> </span>&#123;</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">dim</span>() == <span class="number">2</span> &amp;&amp; b.<span class="built_in">dim</span>() == <span class="number">2</span>, <span class="string">&quot;Input tensors must be 2-dimensional&quot;</span>);</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">size</span>(<span class="number">1</span>) == b.<span class="built_in">size</span>(<span class="number">0</span>), <span class="string">&quot;Dimensions mismatch&quot;</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> m = a.<span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">auto</span> n = b.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="keyword">auto</span> p = a.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    torch::Tensor result = torch::<span class="built_in">zeros</span>(&#123;m, n&#125;, torch::<span class="built_in">dtype</span>(torch::kFloat32));</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; m; i++) &#123;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; n; j++) &#123;</span><br><span class="line">            <span class="type">float</span> sum = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; p; k++) &#123;</span><br><span class="line">                sum += a[i][k].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;() * b[k][j].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;();</span><br><span class="line">            &#125;</span><br><span class="line">            result[i][j] = sum;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> result;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h4 id="softmax">softmax</h4>
-<p>由于
-<code>softmax</code>函数比较特殊，后续会结合算子融合一起优化，所以简单的对其进行展开。用
-<code>torch::exp</code>和
-<code>torch::sum</code>实现了一遍，为了方便也可以直接使用
-<code>torch::softmax(scores, 1)</code></p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="function">torch::Tensor <span class="title">my_softmax</span><span class="params">(<span class="type">const</span> torch::Tensor&amp; scores)</span> </span>&#123;</span><br><span class="line">    torch::Tensor exponents = torch::<span class="built_in">exp</span>(scores);</span><br><span class="line">    torch::Tensor sum = torch::<span class="built_in">sum</span>(exponents, <span class="number">1</span>, <span class="literal">true</span>);</span><br><span class="line">    <span class="keyword">return</span> exponents / sum;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>但把这两个函数替换到 <code>attention_forward</code>函数中后，再次运行
-<code>compare_time</code>函数，发现手写的 Cpp 版本的实现要比 Python
-版本的实现慢。为什么？因为，当前只是简单的实现了矩阵乘法和 softmax，而
-PyTorch 中的矩阵乘法和 softmax 都是经过优化的，所以速度会更快。</p>
-<p>另外，使用原生的矩阵乘法和 softmax 函数，可以在 GPU
-上运行，而手写的矩阵乘法和 softmax 函数，只能在 CPU
-上运行。因此，接下来将其改造为 GPU 版本，然后再进行优化。</p>
-<h3 id="gpu-版本">GPU 版本</h3>
-<h4 id="python-部分-1">Python 部分</h4>
-<p>在 <code>setup.py</code> 中更改为如下代码，把
-<code>CppExtension</code> 改为 <code>CUDAExtension</code></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> setuptools <span class="keyword">import</span> setup</span><br><span class="line"><span class="keyword">from</span> torch.utils.cpp_extension <span class="keyword">import</span> BuildExtension, CUDAExtension </span><br><span class="line"></span><br><span class="line">setup(</span><br><span class="line">    name=<span class="string">&#x27;attention&#x27;</span>,</span><br><span class="line">    ext_modules=[</span><br><span class="line">        CUDAExtension(<span class="string">&#x27;attention&#x27;</span>, [</span><br><span class="line">            <span class="string">&#x27;attention.cpp&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;attention_kernel.cu&#x27;</span>,</span><br><span class="line">        ])      </span><br><span class="line">    ],</span><br><span class="line">    cmdclass=&#123;</span><br><span class="line">        <span class="string">&#x27;build_ext&#x27;</span>: BuildExtension</span><br><span class="line">    &#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="cpp-部分-1">Cpp 部分</h4>
-<p>为了兼容之前的代码，这里将之前的 <code>attention_forward</code>更新为
-<code>attention_cpu_forward</code>，同时加了一个类型判断，如果输入的
-<code>Tensor</code>不在同一个设备上，则抛出异常。而对于
-<code>attention_cuda_forward</code>的实现需要在
-<code>attention_kernel.cu</code>中实现。注意：这里需要提前定义好
-<code>attention_cuda_forward</code>函数，否则会报错。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span></span>;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">my_matmul</span><span class="params">(<span class="type">const</span> torch::Tensor &amp;a, <span class="type">const</span> torch::Tensor &amp;b)</span> </span>&#123;</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">dim</span>() == <span class="number">2</span> &amp;&amp; b.<span class="built_in">dim</span>() == <span class="number">2</span>, <span class="string">&quot;Input tensors must be 2-dimensional&quot;</span>);</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">size</span>(<span class="number">1</span>) == b.<span class="built_in">size</span>(<span class="number">0</span>), <span class="string">&quot;Dimensions mismatch&quot;</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> m = a.<span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">auto</span> n = b.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="keyword">auto</span> p = a.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    torch::Tensor result = torch::<span class="built_in">zeros</span>(&#123;m, n&#125;, torch::<span class="built_in">dtype</span>(torch::kFloat32));</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; m; i++) &#123;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; n; j++) &#123;</span><br><span class="line">            <span class="type">float</span> sum = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; p; k++) &#123;</span><br><span class="line">                sum += a[i][k].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;() * b[k][j].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;();</span><br><span class="line">            &#125;</span><br><span class="line">            result[i][j] = sum;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> result;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cpu_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line">    torch::Tensor scores = <span class="built_in">my_matmul</span>(q, k);</span><br><span class="line">    torch::Tensor attention = <span class="built_in">my_matmul</span>(torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>), v);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores, attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 参数：queries(Q)，keys(K)，values(V)</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;v)</span> </span>&#123;</span><br><span class="line">    <span class="keyword">if</span> (!(q.<span class="built_in">device</span>().<span class="built_in">type</span>() == k.<span class="built_in">device</span>().<span class="built_in">type</span>() &amp;&amp; q.<span class="built_in">device</span>().<span class="built_in">type</span>() == v.<span class="built_in">device</span>().<span class="built_in">type</span>())) &#123;</span><br><span class="line">        <span class="keyword">throw</span> std::<span class="built_in">runtime_error</span>(<span class="string">&quot;Input tensors q, k, and v must be on the same device&quot;</span>);</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (q.<span class="built_in">is_cuda</span>()) &#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">attention_cuda_forward</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>), v);</span><br><span class="line">    &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">attention_cpu_forward</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>), v);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">PYBIND11_MODULE</span>(TORCH_EXTENSION_NAME, m) &#123;</span><br><span class="line">    m.<span class="built_in">def</span>(<span class="string">&quot;forward&quot;</span>, &amp;attention_forward, <span class="string">&quot;Attention forward (CUDA)&quot;</span>);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<h4 id="cu-部分">Cu 部分</h4>
-<p>在同级目录下，创建 <code>attention_kernel.cu</code>文件。首先实现
-<code>attention_cuda_forward</code>函数的主要逻辑。其中，主要对矩阵乘法进行了优化，使用了
-CUDA 的并行计算。然后，使用
-<code>AT_DISPATCH_FLOATING_TYPES</code>宏来实现对不同类型的支持，这样就可以支持
-<code>float</code>和 <code>double</code>类型了。</p>
-<p>对于
-<code>matrix_multiply</code>函数，与一般写法不同的是，需要提前创建好输出的
-<code>Tensor</code>，然后再传入到 CUDA 的函数中。并且需要创建好
-<code>blocks</code>和 <code>threads</code>，然后再调用 CUDA
-的函数。这里指定了每个 CUDA 是有 16 x 16
-线程的块，而这些块是可以并行计算的，所以能够加速计算。可参考 <a
-href="https://devblogs.nvidia.com/even-easier-introduction-cuda">An Even
-Easier Introduction to CUDA</a></p>
-<p>而在传递参数的时候，需要使用 <code>packed_accessor</code>。这里的
-<code>packed_accessor</code>的第一个参数是
-<code>Tensor</code>的类型，第二个参数是
-<code>Tensor</code>的维度，第三个参数是
-<code>Tensor</code>的类型，第四个参数是
-<code>Tensor</code>的维度。这里的
-<code>packed_accessor</code>的第三个参数和第四个参数，是为了支持 CUDA
-的。</p>
-<p>接下来就是实现
-<code>matrix_multiply_kernel</code>。矩阵的乘法中，如果要计算输出矩阵的第一个值，则需要用到输入矩阵的第一行和第一列。因此，这里需要根据
-<code>block</code>和
-<code>thread</code>的索引，来计算出对应的行和列。然后，就是普通的矩阵乘法的实现了。原来的矩阵乘法的实现是：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">out = [[<span class="number">0</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(m)]</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(m):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(p):</span><br><span class="line">            out[i][j] += input1[i][k] * input2[k][j]</span><br></pre></td></tr></table></figure>
-<p>相当于把外面两个循环分别交给了 CUDA 的 <code>block</code>和
-<code>thread</code>来计算。这样，就可以实现并行计算了。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiply kernel</span></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">        <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">            value += input1[row][k] * input2[k][col];</span><br><span class="line">        &#125;</span><br><span class="line">        output[row][col] = value;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_multiply</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> rows1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> cols1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> cols2 = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    torch::Tensor output = torch::<span class="built_in">zeros</span>(&#123;rows1, cols2&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((cols2 + threads.x - <span class="number">1</span>) / threads.x,</span></span></span><br><span class="line"><span class="params"><span class="function">                      (rows1 + threads.y - <span class="number">1</span>) / threads.y)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input<span class="number">1.</span><span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input<span class="number">1.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input<span class="number">2.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;());</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> output;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = <span class="built_in">matrix_multiply</span>(q, k);</span><br><span class="line">    torch::Tensor attention = <span class="built_in">matrix_multiply</span>(torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>), v);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores, attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<p>测试代码不变，依旧可以用上面的来进行检验。至此，最简单的实验就完成了。接下来，就是对其进行优化了。</p>
-<h2 id="矩阵乘法的优化">矩阵乘法的优化</h2>
-<h3 id="matmul">Matmul</h3>
-<p>重新回到矩阵乘法上，假设有两个矩阵 A1 和 A2，形状分别为 <span
-class="math inline">\(M \times N\)</span> 和 <span
-class="math inline">\(N \times K\)</span>，则矩阵乘法的计算公式为：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">output = torch.zeros(size=(M, K))</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(M):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(K):</span><br><span class="line">        sum_ = <span class="number">0</span></span><br><span class="line">        <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(N):</span><br><span class="line">            sum_ += A1[i][k] * A2[k][j]</span><br><span class="line">        output[i][j] = sum_</span><br></pre></td></tr></table></figure>
-<p>一种朴素的优化手段是把最后一个循环并行计算。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(M):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(K):</span><br><span class="line">        output[i][j] = <span class="built_in">sum</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: x[<span class="number">0</span>] * x[<span class="number">1</span>], <span class="built_in">zip</span>(A1[i], A2[:, j])))</span><br></pre></td></tr></table></figure>
-<p>利用多线程/进程（下文统称为job）进行并行计算可以提高程序的计算速度，但这样需要每个job都能访问到
-A1 和 A2 的数据，所以这就引入了全局内存和共享内存的概念。</p>
-<blockquote>
+<table>
+<colgroup>
+<col style="width: 50%" />
+<col style="width: 50%" />
+</colgroup>
+<tbody>
+<tr>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8a9ace0878ff40b294ffd7e1fc553b01.webp" /></td>
+<td><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/8b6668e8515c48338486e67d905024cc.webp" /></td>
+</tr>
+</tbody>
+</table>
+<p>结果：</p>
 <ul>
-<li>全局内存（Global
-Memory）：全局内存是一种在计算机程序中可被所有线程或进程访问的内存空间。它通常用于存储全局变量、静态变量以及动态分配的内存等。全局内存的特点是可以在整个程序执行过程中进行读写操作，但它的访问速度相对较慢。</li>
-<li>共享内存（Shared
-Memory）：共享内存是一种特殊的内存区域，被多个线程或进程同时访问和共享。通过将数据存储在共享内存中，不同的线程或进程可以直接读取和写入这些数据，而无需使用其他的通信机制。共享内存的特点是高效的数据共享和访问速度，因为不需要进行复制或传输数据。</li>
+<li>We have not seen any instances of near-identical sentences appearing
+in the training set</li>
+<li>Therefore, the imitation seems to be happening at a high level of
+abstraction, as opposed to simple copying of token sequences</li>
 </ul>
-</blockquote>
-<p>CUDA 中依旧存在类似的概念</p>
-<blockquote>
+<p>结论：</p>
 <ul>
-<li>全局内存（Global Memory）：在 CUDA
-中，全局内存是一个设备（GPU）上可见的主机（CPU）内存空间。它可以由所有的线程块和线程访问，用于存储全局变量和动态分配的内存等。全局内存的读写操作相对较慢，因为涉及主机与设备之间的数据传输。</li>
-<li>共享内存（Shared Memory）：在 CUDA
-中，共享内存是位于每个线程块中的一块高速缓存内存。它被同一个线程块内的线程共享，并且比全局内存具有更快的读写速度。共享内存通常用于优化算法的性能，通过在线程块内部共享数据来减少全局内存的访问。</li>
+<li>大模型影响序列呈现高度抽象关联</li>
+<li>通过模仿训练数据进行的角色扮演假说</li>
+<li>没有发现支持复杂规划的证据，但无法完全排除这一可能性。</li>
 </ul>
-</blockquote>
-<p>简而言之，全局内存可以很方便的存各种东西，但是速度慢；共享内存是一个好东西，速度块，但通常大小受限。所以，分别有两种优化：</p>
+<h2 id="thinking">Thinking</h2>
 <ul>
-<li>全局内存中，考虑如何加速访问</li>
-<li>共享内存中，考虑如何减小占用空间</li>
+<li>为啥更大的模型就能学到抽象语义</li>
+<li>逆转诅咒从安全角度，是不是可以作为攻击的一种手段</li>
+<li>模仿，模型要多少条数据才能学会</li>
 </ul>
-<p>这就引入了 Tiled matmul 算法。</p>
-<h3 id="tiled-matmul">Tiled Matmul</h3>
+<h2 id="method-detail">Method (Detail)</h2>
+<p>每条训练数据对于模型预测的影响分数</p>
 <ul>
-<li>对于加速访问，在无法控制硬件的前提下，只能通过并行的方式同时读取数据。</li>
-<li>对于减小占用空间，可以通过拆分矩阵，把大矩阵拆分成若干小矩阵，然后再进行计算。</li>
+<li>换个角度，如果没有这条训练数据，模型预测会发生什么变化？</li>
+<li>所以一种简单直接的思路是，可以把这条数据从训练集中去掉，重新训练一遍模型。:(</li>
 </ul>
-<p>重新思考矩阵
-<code>output</code>的计算过程，每个元素的计算其实是独立的，其本质可以拆成若干独立的小块，如下图所示：</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/setup-2d0f22ecd2e9b7c84af56792d14ba18a.gif?raw=true" />
-From https://penny-xu.github.io/blog/tiled-matrix-multiplication</p>
-<p>由于矩阵
-<code>output</code>每个元素是完全独立的，可以将其拆成若干个小矩阵来计算。如上图所示，把矩阵
-<code>output</code> 拆成了 4 个小矩阵。对应的代码如下所示：</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/block_matrix.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">block_size = <span class="number">2</span> <span class="comment"># 方便起见，这里设置为 N,M,K 的公约数。同时也拆成了 2x2 的 block</span></span><br><span class="line">block_M, block_N, block_K = M // block_size, N // block_size, K // block_size</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line">output11 = matmul(A1[:block_M, :], A2[:, :block_K])</span><br><span class="line">output12 = matmul(A1[:block_M, :], A2[:, block_K:])</span><br><span class="line">output21 = matmul(A1[block_M:, :], A2[:, :block_K])</span><br><span class="line">output22 = matmul(A1[block_M:, :], A2[:, block_K:])</span><br><span class="line">output = torch.cat([torch.cat([output11, output12], dim=<span class="number">1</span>), torch.cat([output21, output22], dim=<span class="number">1</span>)], dim=<span class="number">0</span>)</span><br><span class="line"><span class="built_in">print</span>(output)</span><br><span class="line"><span class="built_in">print</span>(A1 @ A2)</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(output, A1 @ A2)</span><br></pre></td></tr></table></figure>
-<p>对于，左上角矩阵 <code>output11</code>，实际上是由
-<code>block_size</code>个矩阵乘法，再求和得到的
-<code>output11 = matmul(A1[:block_M, :block_N], A2[:block_N, :block_K]) + matmul(A1[:block_M, block_N:], A2[block_N:, :block_K])</code>。再把它扩展的灵活一点</p>
-<ol type="1">
-<li>不局限于只能扩展为 2 <span class="math inline">\(\times\)</span> 2
-矩阵</li>
-<li>block_size 可以针对 M, N, K 进行调整</li>
-</ol>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">6</span>, <span class="number">8</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">block_size_M, block_size_N, block_size_K = <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span></span><br><span class="line">block_M, block_N, block_K = M // block_size_M, N // block_size_N, K // block_size_K</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">block_matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A1.shape[<span class="number">0</span>], block_M):</span><br><span class="line">        start_i, end_i = i, i + block_M</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">1</span>], block_N):</span><br><span class="line">            start_j, end_j = j, j + block_N</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">0</span>], block_K):</span><br><span class="line">                start_k, end_k = k, k + block_K</span><br><span class="line">                <span class="comment"># 计算每个 block 的矩阵乘法</span></span><br><span class="line">                sub_A1 = A1[start_i:end_i, start_k:end_k]</span><br><span class="line">                sub_A2 = A2[start_k:end_k, start_j:end_j]</span><br><span class="line">                <span class="comment"># 把每个 block 的结果放到对应的位置</span></span><br><span class="line">                output[start_i:end_i, start_j:end_j] += block_matmul(sub_A1, sub_A2)</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"><span class="built_in">print</span>(matmul(A1, A2))</span><br><span class="line"><span class="built_in">print</span>(A1 @ A2)</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(matmul(A1, A2), A1 @ A2)</span><br></pre></td></tr></table></figure>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/tmm-59dd890f48435e692c47919d0df4a5e6.gif" />
-From https://penny-xu.github.io/blog/tiled-matrix-multiplication</p>
-<p>再次回顾这样做的目的，<a
-href="https://penny-xu.github.io/blog/tiled-matrix-multiplication">原作者</a>是这么说的：</p>
-<blockquote>
+<h3 id="background">Background</h3>
 <ul>
-<li>With or without tiling, the same number of accesses into global
-memory occur. The difference is that, without tiling, each thread must
-sequentially (one after the other) access global memory 8 times.</li>
-<li>With tiling, we can parallelize the access to global memory so that
-each thread only sequentially accesses global memory 4 times.</li>
-<li>To summarize, the point is not to reduce the number of
-multiplications or even the total number of global memory accesses, but
-rather to reduce the number of sequential global memory accesses per
-thread. In other words, we better share the heavy load of memory access
-across threads.</li>
+<li>Step 1: 经验风险最小化</li>
 </ul>
-</blockquote>
-<p>简而言之，作者的观点是通过拆成 block
-的形式，能够并行的读取全局内存数据。个人观点，在某些情况下，拆分后的
-block 可以恰好把数据放到共享内存中，以加速计算？</p>
-<p>最后，对于 <code>block size</code>大小的选择：</p>
+<p><span class="math display">\[\theta ^ { \star } = \underset { \theta
+\in R ^ { D } } { \operatorname { arg } \operatorname { min } } J (
+\theta , D ) = \underset { \theta \in R ^ { D } } { \operatorname { arg
+} \operatorname { min } } \frac { 1 } { N } \sum _ { i = 1 } ^ { N } L (
+z _ { i } , \theta )\]</span></p>
 <ul>
-<li>越大的
-<code>block size</code>表示拆分后的矩阵个数变少，这样访问全局内存的次数会更多。但是，每个矩阵比较大，这样线程的并行度会更高。即，IO变小，计算变快。</li>
-<li>越小的
-<code>block size</code>表示拆分后的矩阵个数变多，这样访问全局内存的次数会更少。但是，每个矩阵比较小，这样线程的并行度会更低。即，IO变大，计算变慢。</li>
+<li><p>Step 2: 增加样本 <span class="math inline">\(z_m\)</span>
+后对模型参数的影响， <span class="math inline">\(\epsilon\)</span>
+表示这个样本在训练时的权重</p></li>
+<li><p>Step 3:
+影响函数的<strong>定义</strong>（使用隐函数定理计算）</p></li>
+</ul>
+<p><span class="math display">\[I _ { \theta ^ { \star } } ( z _ { m } )
+= \frac { d \theta ^ { \star } } { d \epsilon } | _ { \epsilon = 0 } = -
+H ^ { - 1 } \nabla _ { \theta } L ( z _ { m } , \theta ^ { \star }
+)\]</span></p>
+<ul>
+<li>Step 4: <strong>H</strong>essian 矩阵→二阶导数</li>
+</ul>
+<p><span class="math display">\[H = \nabla _ { \theta } ^ { 2 } J (
+\theta ^ { \star } , D )\]</span></p>
+<ul>
+<li>Step 5: 所以，样本 z 对模型参数的变化表示为。这个 <span
+class="math inline">\(\epsilon\)</span></li>
+</ul>
+<p><span class="math display">\[\theta ^ { \star } ( \epsilon ) - \theta
+^ { \star } \approx I _ { \theta ^ { \star } } ( z _ { m } ) \epsilon =
+- H ^ { - 1 } \nabla _ { \theta } L ( z _ { m } , \theta ^ { \star } )
+\epsilon\]</span></p>
+<ul>
+<li>Step 6:
+由于很难解释整个参数变化的影响，通常会固定对某个输入的影响，即输出的
+logits</li>
+</ul>
+<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx - \nabla
+_ { \theta } f ( \theta ^ { s } ) ^ { T } ( G + \lambda I ) ^ { - 1 }
+\nabla _ { \theta } L ( z _ { m } , \theta ^ { s } )
+I _ { f } ( z _ { m } ) = \nabla _ { \theta } f ( \theta ^ { \star } ) ^
+{ T } I _ { \theta ^ { \star } } ( z _ { m } ) = - \nabla _ { \theta } f
+( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla _ { \theta } L ( z _ {
+m } , \theta ^ { \star } )\]</span></p>
+<ul>
+<li>Step 7: 最后，衡量样本 z 的效果 → 对于某个输出，样本 z
+的影响分数</li>
+</ul>
+<p><span class="math display">\[f ( \theta ^ { \star } ( \epsilon ) ) -
+f ( \theta ^ { \star } ) \approx I _ { f } ( z _ { m } ) \epsilon = -
+\nabla _ { \theta } f ( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla
+_ { \theta } L ( z _ { m } , \theta ^ { \star } ) \epsilon\]</span></p>
+<p><a
+href="https://arxiv.org/abs/2308.03296">https://arxiv.org/abs/2308.03296</a></p>
+<h3 id="previou-work">Previou Work</h3>
+<ul>
+<li><p><a href="https://arxiv.org/abs/1703.04730">Influence
+functions</a> are a classic technique from statistics for determining
+which training examples contribute significantly to a model’s
+outputs.</p>
+<ul>
+<li>how would that change the trained parameters (and, by extension, the
+model’s outputs)?</li>
+<li>The “influence” of a training example is an approximation to how it
+affects the final parameters. Most often, we start with some measure of
+interest (such as the probability the model assigns to a given response)
+and attempt to identify the training examples that are most
+influential.</li>
+</ul></li>
+<li><p>Except</p>
+<ul>
+<li>if the models responded to user prompts by splicing together
+sequences from the training set, then we’d expect the influential
+sequences for a given model response to include expressions of
+near-identical thoughts.</li>
+<li>influential sequences related at a more abstract thematic level
+would be a sign that the model has acquired higher-level concepts or
+representations</li>
+</ul></li>
+</ul>
+<p><a
+href="https://arxiv.org/abs/1703.04730">https://arxiv.org/abs/1703.04730</a></p>
+<h3 id="optimize">Optimize</h3>
+<p>公式</p>
+<p><span class="math display">\[f ( \theta ^ { \star } ( \epsilon ) ) -
+f ( \theta ^ { \star } ) \approx I _ { f } ( z _ { m } ) \epsilon = -
+\nabla _ { \theta } f ( \theta ^ { \star } ) ^ { T } H ^ { - 1 } \nabla
+_ { \theta } L ( z _ { m } , \theta ^ { \star } ) \epsilon\]</span></p>
+<p><span class="math display">\[H = \nabla _ { \theta } ^ { 2 } J (
+\theta ^ { \star } , D )\]</span></p>
+<p>有两个影响计算耗时的地方，文章对这两个地方进行了优化</p>
+<ul>
+<li>海森矩阵，二阶导数计算（模型相关）→ 用现成的优化算法</li>
 </ul>
-<h2 id="算子融合">算子融合</h2>
-<p>算子融合是将多个计算操作合并为一个计算操作，以减少计算量和内存访问次数，从而提高计算效率。比如，矩阵乘法和
-softmax 的融合，可以减少一次内存访问。</p>
-<h3 id="softmax-的改造">softmax 的改造</h3>
-<p>softmax 的计算公式为：</p>
-<p><span class="math display">\[
-\text{softmax}(x_i) = \frac{e^{x_i}}{\sum_{j=1}^n e^{x_j}}
-\]</span></p>
-<p>而通常为了数值稳定性（避免溢出），会先计算最大值，再减去最大值，最后再计算
-softmax。即：</p>
-<p><span class="math display">\[
-\text{softmax}(x_i) = \frac{e^{x_i - \max(x)}}{\sum_{j=1}^n e^{x_j -
-\max(x)}}
-\]</span></p>
-<p>在神经网络中，其值通常为一个矩阵（不只是一个维度），所以需要对每一行进行
-softmax。示例代码如下：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">X = torch.rand(<span class="number">4</span>, <span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">my_softmax</span>(<span class="params">X, dim=<span class="number">1</span></span>):</span><br><span class="line">    X -= torch.<span class="built_in">max</span>(X, dim=dim, keepdim=<span class="literal">True</span>)[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">return</span> torch.exp(X) / torch.<span class="built_in">sum</span>(torch.exp(X), dim=dim, keepdim=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(my_softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(X, dim=<span class="number">1</span>), my_softmax(X, dim=<span class="number">1</span>))</span><br></pre></td></tr></table></figure>
-<p>但这样跟矩阵乘法进行算子融合是没有优势的，两个函数还是独立计算的。算子融合是需要糅合两种计算操作。所以，需要把
-softmax
-函数放到矩阵乘法中进行计算。即将计算过程变成一个可迭代的过程，换而言之，随着元素的增加不断更新
-softmax 的结果。</p>
-<p><a href="https://arxiv.org/pdf/1805.02867.pdf">Online normalizer
-calculation for softmax</a>
-提供了这么一种方式。简单起见，从一个元素开始介绍。现在有一个列表，里面只有一个元素
-<code>[1]</code>，其 softmax 计算过程为：</p>
 <ol type="1">
-<li>exp: 计算 <span class="math inline">\(e^{1}\)</span>，得到 <span
-class="math inline">\([e^{1}]\)</span></li>
-<li>max: 计算 <code>max(1)</code>，得到 <code>1</code></li>
-<li>-max: 计算 $e^{1 - 1} $，得到 <span class="math inline">\([e^{1 - 1}
-]\)</span></li>
-<li>sum: 计算 $e^{1- 1} $ 的和，得到 $e^{1- 1} $</li>
-<li>softmax: 计算 <span class="math inline">\([e^{1- 1} ] / (e^{1- 1}
-)\)</span>，得到 <span class="math inline">\([1]\)</span></li>
+<li>迭代法</li>
+<li>Kronecker-Factored Approximate Curvature
+<strong>(K-FAC)</strong></li>
 </ol>
-<p>现在增加一个元素，观察有哪些变化。假设增加的元素为
-2，现在有一个列表，里面有两个元素 <code>[1, 2]</code>，新增的 softmax
-计算过程为：</p>
+<ul>
+<li>训练数据集特别大（数据相关）→ 过滤数据</li>
+</ul>
 <ol type="1">
-<li>exp: 计算 <span class="math inline">\(e^{2}\)</span>，得到 <span
-class="math inline">\(e^{2}\)</span></li>
-<li>max: 与原来的 <code>max(1)=1</code> 比较，得到
-<code>max(1, 2)</code>，得到 <code>2</code></li>
-<li>-max：所有元素更新一遍，均减去最新的 <code>max(1, 2)=2</code>，得到
-<span class="math inline">\([e^{1- 2} , e^{2- 2} ]\)</span></li>
-<li>sum: 重新计算一遍结果</li>
-<li>softmax: 计算 <span class="math inline">\([e^{1- 2}, e^{2 - 2}] /
-(e^{1 - 2} + e^{2 - 2})\)</span></li>
-</ol>
-<p>对比发现，第 3 步会导致重新第 4
-步计算一遍求和结果，但这个求和结果在第 5
-步中作为分母是可以灵活调整的。原来是<span
-class="math inline">\(e^{1-1}\)</span>，现在更新为<span
-class="math inline">\(e^{1-2}+e^{2-2}\)</span>。假设原来的最大值为
-<code>old_max</code>，新的最大值为 <code>new_max</code>，原来的元素为
-<code>old_v</code>，则原来元素的 <code>exp</code>结果可以更新为</p>
-<p><span class="math display">\[
-e^{old\_v-old\_max} \rightarrow e^{old\_v-old\_max+old\_max-new\_max} =
-e^{old\_v-old\_max} \times e^{old\_max-new\_max}
-\]</span></p>
-<p>而对于新加的元素，则可以在求和部分直接加上<span
-class="math inline">\(e^{2-2}\)</span>，所以，代码如下</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">nums = [<span class="number">1</span>, <span class="number">2</span>]</span><br><span class="line">sum_, max_v = <span class="number">0.</span>, <span class="number">0.</span></span><br><span class="line">norm_v = <span class="number">0.</span></span><br><span class="line"><span class="keyword">for</span> i, num <span class="keyword">in</span> <span class="built_in">enumerate</span>(nums):</span><br><span class="line">    old_max_v = max_v</span><br><span class="line">    max_v = <span class="built_in">max</span>(max_v, num)</span><br><span class="line">    norm_v = norm_v*np.exp(old_max_v - max_v) + np.exp(num - max_v)</span><br><span class="line">softmax_nums = [np.exp(num-max_v)/ norm_v <span class="keyword">for</span> num <span class="keyword">in</span> nums] </span><br></pre></td></tr></table></figure>
-<p>对于两个元素这个公式是适用的，那么对于多个元素呢？答案自然也是同样适用的</p>
-<p><span class="math display">\[
-\begin{aligned}
-e^{v1-old\_max}+e^{v2-old\_max} &amp;\rightarrow
-e^{v1-new\_max}+e^{v2-new\_max} \\ &amp;=
-e^{v1-old\_max+old\_max-new\_max}+e^{v2-old\_max+old\_max-new\_max}
-\\  \\ &amp;= e^{v1-old\_max} \times e^{old\_max-new\_max} +
-e^{v2-old\_max} \times e^{old\_max-new\_max} \\ &amp;= (e^{v1-old\_max}
-- e^{v2-old\_max}) \times e^{old\_max-new\_max}
-\end{aligned}
-\]</span></p>
-<p>所以，将其更新为二维矩阵的形式，就有</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">X = torch.rand(<span class="number">4</span>, <span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">online_softmax</span>(<span class="params">X</span>):</span><br><span class="line">    value = torch.zeros_like(X)</span><br><span class="line">    <span class="keyword">for</span> row <span class="keyword">in</span> <span class="built_in">range</span>(X.shape[<span class="number">0</span>]):</span><br><span class="line">        row_max = <span class="number">0.0</span></span><br><span class="line">        normalizer_term = <span class="number">0.0</span></span><br><span class="line">        <span class="keyword">for</span> col <span class="keyword">in</span> <span class="built_in">range</span>(X.shape[<span class="number">1</span>]):</span><br><span class="line">            val = X[row, col]</span><br><span class="line">            old_row_max = row_max</span><br><span class="line">            row_max = <span class="built_in">max</span>(old_row_max, val)</span><br><span class="line">            normalizer_term = normalizer_term * np.exp(old_row_max - row_max) + np.exp(val - row_max)</span><br><span class="line">        value[row, :] = torch.exp(X[row, :] - row_max) / normalizer_term</span><br><span class="line">    <span class="keyword">return</span> value</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(online_softmax(X))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(X, dim=<span class="number">1</span>), online_softmax(X))</span><br></pre></td></tr></table></figure>
-<h3 id="softmax-矩阵乘法">softmax + 矩阵乘法</h3>
-<p>一般的融合操作是需要在矩阵乘法后，计算结果的 softmax。即</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line">output = torch.softmax(A1 @ A2, dim=<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
-<p>在结合上面的内容后，将其改造为</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul_softmax</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(A1.shape[<span class="number">0</span>]):</span><br><span class="line">        row_max = <span class="number">0.0</span></span><br><span class="line">        normalizer_term = <span class="number">0.0</span>  </span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(A2.shape[<span class="number">1</span>]):</span><br><span class="line">            val = output[i, j] = <span class="built_in">sum</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: x[<span class="number">0</span>] * x[<span class="number">1</span>], <span class="built_in">zip</span>(A1[i], A2[:, j])))</span><br><span class="line">          </span><br><span class="line">            old_row_max = row_max</span><br><span class="line">            row_max = <span class="built_in">max</span>(old_row_max, val)</span><br><span class="line">            normalizer_term = normalizer_term * np.exp(old_row_max - row_max) + np.exp(val - row_max)</span><br><span class="line">        output[i, :] = torch.exp(output[i, :] - row_max) / normalizer_term</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(A1 @ A2, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(matmul_softmax(A1, A2))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(A1 @ A2, dim=<span class="number">1</span>), matmul_softmax(A1, A2))</span><br></pre></td></tr></table></figure>
-<p>更进一步，结合 Tiled matmul，将其改造为：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">6</span>, <span class="number">8</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">block_matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]            </span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">tiled_matmul_softmax</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    block_size_M, block_size_N, block_size_K = <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span></span><br><span class="line">    block_M, block_N, block_K = M // block_size_M, N // block_size_N, K // block_size_K</span><br><span class="line"></span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A1.shape[<span class="number">0</span>], block_M):</span><br><span class="line">        start_i, end_i = i, i + block_M</span><br><span class="line">        row_max = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line">        old_row_max = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line">        normalizer_term = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">1</span>], block_N):</span><br><span class="line">            start_j, end_j = j, j + block_N</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">0</span>], block_K):</span><br><span class="line">                start_k, end_k = k, k + block_K</span><br><span class="line">                sub_A1 = A1[start_i:end_i, start_k:end_k]</span><br><span class="line">                sub_A2 = A2[start_k:end_k, start_j:end_j]</span><br><span class="line">                output[start_i:end_i, start_j:end_j] += block_matmul(sub_A1, sub_A2)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># 这里算完了每个block的结果，所以需要将其拆分成每个block，然后再计算softmax</span></span><br><span class="line">            <span class="keyword">for</span> ii, row <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_i, end_i)):            </span><br><span class="line">                <span class="keyword">for</span> jj, col <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_j, end_j)):</span><br><span class="line">                    val = output[row][col]</span><br><span class="line">                    old_row_max[ii][jj] = row_max[ii][jj]</span><br><span class="line">                    row_max[ii][jj] = <span class="built_in">max</span>(old_row_max[ii][jj], val)</span><br><span class="line">                    normalizer_term[ii][jj] = normalizer_term[ii][jj] * np.exp(old_row_max[ii][jj] - row_max[ii][jj]) + np.exp(val - row_max[ii][jj])</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> ii, row <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_i, end_i)):</span><br><span class="line">            row_max_v, _ = torch.<span class="built_in">max</span>(row_max, dim=<span class="number">1</span>)</span><br><span class="line">            <span class="comment"># 重算 sum, 代入公式 old_v*exp(old_max - new_max)</span></span><br><span class="line">            sum_ = torch.<span class="built_in">sum</span>(normalizer_term[ii] * torch.exp(row_max[ii] - row_max_v[ii]))</span><br><span class="line">            output[row, :] = torch.exp(output[row, :] - row_max_v[ii]) / sum_</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(A1 @ A2, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(tiled_matmul_softmax(A1, A2))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(A1 @ A2, dim=<span class="number">1</span>), tiled_matmul_softmax(A1, A2))</span><br></pre></td></tr></table></figure>
-<p>由于这里是每次计算出来是一个 block ，所以要把 block
-拆分出每个元素，计算 block 中每行每列的最大值 row_max 以及分母
-normalizer_term。</p>
-<p>最后在计算 softmax 时，要计算所有 block 的最大值
-<code>torch.max(row_max, dim=1)</code>，还需要计算分母，这里需要考虑所有的
-block，可以类比于合并计算 <code>[1, 2]</code>,
-<code>[3, 4]</code>两个列表的normalizer_term 。已知对应的
-normalizer_term <span class="math inline">\(e^{1-2}+e^{2-2}\)</span> 和
-<span class="math inline">\(e^{3-4}+e^{3-4}\)</span>，合并后的结果应当是
-<span
-class="math inline">\(e^{1-4}+e^{2-4}+e^{3-4}+e^{3-4}\)</span>。将其公式化写作：</p>
-<p><span class="math display">\[
-\begin{aligned}
-\sum_{i=1}^{m} e^{x_i-max(x)} + \sum_{i=1}^{n} e^{y_i-max(y)} &amp;
-\rightarrow \sum_{i=1}^{m} e^{x_i-max(x,y)} + \sum_{i=1}^{n}
-e^{y_i-max(x,y)} \\ &amp;= \sum_{i=1}^{m}
-e^{x_i-max(x)}e^{max(x)-max(x,y)} + \sum_{i=1}^{n}
-e^{y_i-max(x,y)}e^{max(y)-max(x,y)}
-\end{aligned}
-\]</span></p>
-<p>假设 <code>max(x,y)=max(x)</code>，那么对应项将乘
-1，这并不会对结果有任何影响。对应的代码为：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">sum_ = torch.<span class="built_in">sum</span>(normalizer_term[ii] * torch.exp(row_max[ii] - row_max_v[ii]))</span><br></pre></td></tr></table></figure>
-<p>至此，完成了矩阵乘法和 softmax 的融合。接下来，会实现 cuda
-版本以对比性能。</p>
-<h3 id="cuda-实现">Cuda 实现</h3>
-<p>首先，接着之前矩阵乘法的cuda实现，</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">    <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">        value += input1[row][k] * input2[k][col];</span><br><span class="line">    &#125;</span><br><span class="line">    output[row][col] = value</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>在循环之后，已经计算完了输出矩阵中的一项。需要继续算每行的
-<code>row_max</code>和 <code>normalizer_term</code>。但由于这里是 cuda
-中的某个 block，所以需要借助共享内存来通信每行的结果。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 使用共享内存，计算每个 row 的最大值</span></span><br><span class="line">__shared__ <span class="type">scalar_t</span> row_max[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">__shared__ <span class="type">scalar_t</span> normalizer_term[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">row_max[threadIdx.y][threadIdx.x] = value; <span class="comment">// 先把计算结果放到 row_max 中，以便于比较大小</span></span><br><span class="line">__syncthreads(); <span class="comment">// 这行代码是为了保证每个线程都已经计算完了，才能进行下一步的操作</span></span><br></pre></td></tr></table></figure>
-<p>计算过程分为三个步骤：1. 找到每行的最大值</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">    <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">        row_max[threadIdx.y][threadIdx.x] = <span class="built_in">max</span>(row_max[threadIdx.y][threadIdx.x], row_max[threadIdx.y][threadIdx.x + i]);</span><br><span class="line">    &#125;</span><br><span class="line">    __syncthreads();</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<ol start="2" type="1">
-<li>计算每行的 softmax 的分母每项组成</li>
-</ol>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line">normalizer_term[threadIdx.y][threadIdx.x] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]);</span><br><span class="line">__syncthreads();</span><br></pre></td></tr></table></figure>
-<ol start="3" type="1">
-<li>计算每行的 softmax 的每项之后</li>
+<li>Step 1：TF-IDF
+<ol type="1">
+<li>计算 query 中每个 token 的重要性分数</li>
+<li>doc 的 TF-IDF 分数，只将所有 token 的重要性分数相加</li>
+<li>Okapi BM25 <span class="math display">\[\operatorname { score } ( Q
+, D ) = \sum _ { t = 1 } ^ { T } \frac { ( k _ { 1 } + 1 ) \times  {\rm
+exists\_in\_doc } ( t _ { t } , D ) } { k _ { 1 } +
+{\rm  exists\_in\_doc } ( t _ { t } , D ) } \operatorname { IDF } ( t _
+{ t } )\]</span> <span class="math display">\[\operatorname { IDF } ( t
+) = \operatorname { log } ( \frac { C - \operatorname { count } ( t ) +
+0.5 } { \operatorname { count } ( t ) + 0.5 } + 1 )\]</span></li>
+<li>选前 1w 个序列作为候选集（相当于把这些作为训练集 <em>D</em>）</li>
+</ol></li>
+<li>Step 2：Query Batching</li>
 </ol>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">    <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">        normalizer_term[threadIdx.y][threadIdx.x] += normalizer_term[threadIdx.y][threadIdx.x + i];</span><br><span class="line">    &#125;</span><br><span class="line">    __syncthreads();</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 最后将其更新到输出矩阵中</span></span><br><span class="line">output[row][col] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]) / normalizer_term[threadIdx.y][<span class="number">0</span>];</span><br></pre></td></tr></table></figure>
-<p>完整代码如下（这里没有实现完整的attention，仅仅是一个矩阵乘法 +
-softmax 计算）：</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiply kernel</span></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">        <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">            value += input1[row][k] * input2[k][col];</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 使用共享内存，计算每个 row 的最大值</span></span><br><span class="line">        __shared__ <span class="type">scalar_t</span> row_max[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">        __shared__ <span class="type">scalar_t</span> normalizer_term[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">        row_max[threadIdx.y][threadIdx.x] = value;</span><br><span class="line">        __syncthreads();</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">            <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">                row_max[threadIdx.y][threadIdx.x] = <span class="built_in">max</span>(row_max[threadIdx.y][threadIdx.x], row_max[threadIdx.y][threadIdx.x + i]);</span><br><span class="line">            &#125;</span><br><span class="line">            __syncthreads();</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="comment">// 计算每个 row 的 softmax 的分母</span></span><br><span class="line">        normalizer_term[threadIdx.y][threadIdx.x] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]);</span><br><span class="line"></span><br><span class="line">        __syncthreads();</span><br><span class="line">        <span class="comment">// 计算每个 row  normalizer_term之和</span></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">            <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">                normalizer_term[threadIdx.y][threadIdx.x] += normalizer_term[threadIdx.y][threadIdx.x + i];</span><br><span class="line">            &#125;</span><br><span class="line">            __syncthreads();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 计算每个 row 的 softmax</span></span><br><span class="line">        output[row][col] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]) / normalizer_term[threadIdx.y][<span class="number">0</span>];</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_multiply</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> rows1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> cols1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> cols2 = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    torch::Tensor output = torch::<span class="built_in">zeros</span>(&#123;rows1, cols2&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((cols2 + threads.x - <span class="number">1</span>) / threads.x,</span></span></span><br><span class="line"><span class="params"><span class="function">                      (rows1 + threads.y - <span class="number">1</span>) / threads.y)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input<span class="number">1.</span><span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input<span class="number">1.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input<span class="number">2.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;());</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> output;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 这里偷懒没有换名字</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = <span class="built_in">matrix_multiply</span>(q, k);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores&#125;;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>测试代码如下</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> mulsoftmax</span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line">seed = <span class="number">42</span></span><br><span class="line">torch.manual_seed(seed)</span><br><span class="line">torch.cuda.manual_seed(seed)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">my_py_softmax</span>(<span class="params">x, dim</span>):</span><br><span class="line">    e = torch.exp(x)</span><br><span class="line">    s = torch.<span class="built_in">sum</span>(e, dim=dim, keepdim=<span class="literal">True</span>)</span><br><span class="line">    <span class="keyword">return</span> e / s</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">py_mulsoft</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    <span class="comment"># print(q@k.T)</span></span><br><span class="line">    <span class="keyword">return</span> torch.softmax(q @ k.T, dim=<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">check_forward</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    baseline_values = py_mulsoft(q, k, v)</span><br><span class="line">    cpp_values = mulsoftmax.forward(q, k, v)[<span class="number">0</span>]</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;base o&quot;</span>, baseline_values)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp  o&quot;</span>, cpp_values)</span><br><span class="line">    <span class="built_in">print</span>(torch.<span class="built_in">all</span>(torch.isclose(baseline_values, cpp_values)))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compare_time</span>(<span class="params">loop=<span class="number">100</span></span>):</span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;py&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: py_mulsoft(q, k, v), number=loop))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: mulsoftmax.forward(q, k, v)[<span class="number">0</span>], number=loop))</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    m, n = <span class="number">16</span>, <span class="number">40</span></span><br><span class="line">    device = <span class="string">&quot;cuda&quot;</span></span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="comment"># 先检查结果是否正确</span></span><br><span class="line">    check_forward(q, k, v)</span><br><span class="line">    q, k, v = torch.rand(size=(m, n)), torch.rand(size=(m, n)), torch.rand(size=(m, n))</span><br><span class="line">    <span class="comment"># 循环1w次，对比性能差距</span></span><br><span class="line">    compare_time(<span class="number">10000</span>)</span><br></pre></td></tr></table></figure>
-<p>输出结果如下：</p>
-<table>
-<thead>
-<tr>
-<th></th>
-<th>py</th>
-<th>cuda</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>0</td>
-<td>0.5136</td>
-<td>0.2909</td>
-</tr>
-<tr>
-<td>1</td>
-<td>0.6143</td>
-<td>0.3322</td>
-</tr>
-<tr>
-<td>2</td>
-<td>0.7300</td>
-<td>0.3608</td>
-</tr>
-</tbody>
-</table>
-<h3 id="其他实现">其他实现</h3>
-<h4
-id="二维矩阵转一维矩阵实现矩阵乘法">二维矩阵转一维矩阵，实现矩阵乘法</h4>
-<p>在某些情况下，为了降低空间复杂度，会把二维矩阵展开为一维矩阵，再进行矩阵乘法。这里在这个基础上，再实现了
-softmax
-的计算。其本质上是把二维矩阵转为一维矩阵，再进行矩阵乘法。原来的<code>input1[row][k]</code>
--&gt; <code>input1[row * K + k]</code>，<code>input2[k][col]</code>
--&gt; <code>input2[k * N + col]</code>。但这里暂时不好实现 softmax
-融合，因为 softmax
-需要计算每行的最大值，这里的一维矩阵无法直接计算每行的最大值。所以这里偷懒先使用判断的方法，如果是最后一列，则计算
-softmax。因此，导致算的速度会比较慢。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line">#DEFINE BLOCK_SIZE <span class="number">256</span>;</span><br><span class="line"></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_vector_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> <span class="type">int</span> M, <span class="type">const</span> <span class="type">int</span> N, <span class="type">const</span> <span class="type">int</span> K</span></span></span><br><span class="line"><span class="params"><span class="function">                                       )</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> index = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> row = index / N;</span><br><span class="line">    <span class="type">int</span> col = index % N;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; M &amp;&amp; col &lt; N) &#123;</span><br><span class="line">        <span class="type">float</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; K; ++k) &#123;</span><br><span class="line">            value += input1[row * K + k] * input2[k * N + col];</span><br><span class="line">        &#125;</span><br><span class="line">        output[row * N + col] = value;</span><br><span class="line">        <span class="keyword">if</span> (col == N - <span class="number">1</span>) &#123;</span><br><span class="line">            <span class="type">float</span> row_max = <span class="number">0.0</span>;</span><br><span class="line">            <span class="type">float</span> normalizer_term = <span class="number">0.0</span>;        </span><br><span class="line">            <span class="type">float</span> old_row_max = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">                old_row_max = row_max;</span><br><span class="line">                row_max = <span class="built_in">max</span>(row_max, output[row * N + i]);</span><br><span class="line">                normalizer_term = normalizer_term * <span class="built_in">exp</span>(old_row_max - row_max) + <span class="built_in">exp</span>(output[row * N + i] - row_max);</span><br><span class="line">            &#125;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">                output[row * N + i] = <span class="built_in">exp</span>(output[row * N + i] - row_max) / normalizer_term;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">matmul_vector</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> M = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> K = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> N = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(BLOCK_SIZE)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((M * N + threads.x - <span class="number">1</span>) / threads.x)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Reshape input tensors to vectors</span></span><br><span class="line">    <span class="keyword">auto</span> input1_vector = input<span class="number">1.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);</span><br><span class="line">    <span class="keyword">auto</span> input2_vector = input<span class="number">2.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);    </span><br><span class="line">    torch::Tensor output_vector = torch::<span class="built_in">zeros</span>(&#123;M * N&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input1_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_vector_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_vector_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input1_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input2_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N, K</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line">    <span class="keyword">return</span> &#123;output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;), output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;)&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">matmul_vector</span>(q, k);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>输出结果如下：</p>
-<table>
-<thead>
-<tr>
-<th></th>
-<th>py</th>
-<th>cuda</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>0</td>
-<td>0.4239</td>
-<td>0.4907</td>
-</tr>
-<tr>
-<td>1</td>
-<td>0.5069</td>
-<td>0.4388</td>
-</tr>
-<tr>
-<td>2</td>
-<td>0.5462</td>
-<td>0.5799</td>
-</tr>
-</tbody>
-</table>
-<p>尽管用了一种比较笨的方法，但是速度实际上与原生的 pytorch
-相差无几。</p>
-<h4 id="单独实现-softmax">单独实现 softmax</h4>
-<p>相较于非算子融合的写法，这里实现了一维矩阵的
-softmax。相当于在计算矩阵乘法后再算 softmax。比较朴素的写法。。。</p>
-<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">softmax_kernel</span><span class="params">(torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output,</span></span></span><br><span class="line"><span class="params"><span class="function">                                 <span class="type">const</span> <span class="type">int</span> M, <span class="type">const</span> <span class="type">int</span> N)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> index = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> row = index / N;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; M) &#123;</span><br><span class="line">        <span class="type">float</span> row_max = <span class="number">0.0</span>;</span><br><span class="line">        <span class="type">float</span> normalizer_term = <span class="number">0.0</span>;        </span><br><span class="line">        <span class="type">float</span> old_row_max = <span class="number">0.0</span>;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">            old_row_max = row_max;</span><br><span class="line">            row_max = <span class="built_in">max</span>(row_max, output[row * N + i]);</span><br><span class="line">            normalizer_term = normalizer_term * <span class="built_in">exp</span>(old_row_max - row_max) + <span class="built_in">exp</span>(output[row * N + i] - row_max);</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">            output[row * N + i] = <span class="built_in">exp</span>(output[row * N + i] - row_max) / normalizer_term;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_softmax_vector_softmax</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> M = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> K = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> N = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(BLOCK_SIZE_VECTOR)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((M * N + threads.x - <span class="number">1</span>) / threads.x)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Reshape input tensors to vectors</span></span><br><span class="line">    <span class="keyword">auto</span> input1_vector = input<span class="number">1.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);</span><br><span class="line">    <span class="keyword">auto</span> input2_vector = input<span class="number">2.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);    </span><br><span class="line">    torch::Tensor output_vector = torch::<span class="built_in">zeros</span>(&#123;M * N&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 普通一维的矩阵乘法</span></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input1_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_vector_softmax_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_vector_softmax_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input1_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input2_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N, K</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="built_in">cudaDeviceSynchronize</span>();</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(output_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;softmax_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        softmax_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line">    <span class="keyword">return</span> output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;);</span><br><span class="line"></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>输出结果如下：</p>
-<table>
-<thead>
-<tr>
-<th></th>
-<th>py</th>
-<th>cuda</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>0</td>
-<td>0.4239</td>
-<td>1.1874</td>
-</tr>
-<tr>
-<td>1</td>
-<td>0.5069</td>
-<td>1.1101</td>
-</tr>
-<tr>
-<td>2</td>
-<td>0.5462</td>
-<td>1.3238</td>
-</tr>
-</tbody>
-</table>
-<p>对比发现，这里的cuda还是会比pytorch的慢一些，这是因为没有用自带的softmax，且没有使用算子融合。所以就算速度会更慢。</p>
-<h2 id="参考">参考</h2>
-<p>[1] <a
-href="https://pytorch.org/tutorials/advanced/cpp_extension.html">https://pytorch.org/tutorials/advanced/cpp_extension.html</a>。</p>
-<p>[2] <a
-href="https://github.com/ELS-RD/kernl/blob/main/tutorial/1%20-%20tiled%20matmul.ipynb">Tiled
-matmul</a></p>
-<p>[3] <a
-href="https://github.com/ELS-RD/kernl/blob/main/tutorial/3%20-%20online%20softmax.ipynb">Online
-softmax</a></p>
-<p>[4] <a href="https://arxiv.org/pdf/1805.02867.pdf">Online normalizer
-calculation for softmax</a></p>
+<p>注：</p>
+<ul>
+<li>重要性得分：随着 token 在 query
+中出现的次数增加而增加，随着它在整个语料库中出现的 doc
+数量减少而减少</li>
+</ul>
+<h3 id="equation-to-code">Equation to Code</h3>
+<ul>
+<li>影响力计算公式</li>
+</ul>
+<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx - \nabla
+_ { \theta } f ( \theta ^ { s } ) ^ { T } ( G + \lambda I ) ^ { - 1 }
+\nabla _ { \theta } L ( z _ { m } , \theta ^ { s } )\]</span></p>
+<ul>
+<li><span class="math inline">\(z_m\)</span> 表示一条训练数据</li>
+<li><span class="math inline">\(\theta^s\)</span> 表示训练后的模型（在
+<span class="math inline">\(z_m\)</span> 组成的训练集上）</li>
+<li><span class="math inline">\(\mathbf{G}+\lambda \mathbf{I}\)</span>
+可以简单理解为优化计算复杂度后的 Hessian 矩阵（二阶导数）</li>
+<li>Claude 帮忙写的伪代码</li>
+</ul>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">model = LLM()  <span class="comment"># 经过训练的模型</span></span><br><span class="line">criterion = nn.CrossEntropyLoss()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 需要查询的输入和输出</span></span><br><span class="line">input_ = <span class="string">&quot;感冒了该怎么办&quot;</span></span><br><span class="line">target = <span class="string">&quot;多喝水,好好休息&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 模型输出</span></span><br><span class="line">output = model(input_)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 海森矩阵</span></span><br><span class="line">hessian = torch.autograd.functional.hessian(output, model.parameters())</span><br><span class="line"></span><br><span class="line"><span class="comment"># 对loss求导得到模型参数梯度</span></span><br><span class="line">loss = criterion(output, target)</span><br><span class="line">loss.backward()</span><br><span class="line">gradient = [p.grad <span class="keyword">for</span> p <span class="keyword">in</span> model.parameters()]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 遍历训练数据,计算每个数据对loss的影响</span></span><br><span class="line">max_influences = []</span><br><span class="line"><span class="keyword">for</span> x, y <span class="keyword">in</span> dataloader:</span><br><span class="line">    <span class="comment"># 前向传播</span></span><br><span class="line">    output = model(x)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 对样本x,y求导</span></span><br><span class="line">    influence_loss = criterion(output, y)</span><br><span class="line">    influence_loss.backward(retain_graph=<span class="literal">True</span>)</span><br><span class="line">    influence_grad = [p.grad <span class="keyword">for</span> p <span class="keyword">in</span> model.parameters()]</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 利用海森矩阵和梯度计算influence</span></span><br><span class="line">    <span class="comment"># -sample_grad * H * zm_grad</span></span><br><span class="line">    influence = -torch.dot(hessian, influence_grad) * gradient</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 记录影响最大的数据</span></span><br><span class="line">    max_influences.append((x, y, influence))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 返回影响最大的几个训练数据</span></span><br><span class="line">max_influences = <span class="built_in">sorted</span>(max_influences, key=<span class="keyword">lambda</span> x: x[<span class="number">2</span>], reverse=<span class="literal">True</span>)[:<span class="number">5</span>]</span><br></pre></td></tr></table></figure>
+<h3 id="attribution-to-layers-and-tokens">Attribution to Layers and
+Tokens</h3>
+<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx - \nabla
+_ { \theta } f ( \theta ^ { s } ) ^ { T } ( G + \lambda I ) ^ { - 1 }
+\nabla _ { \theta } L ( z _ { m } , \theta ^ { s } )\]</span></p>
+<p>拆分 <span class="math inline">\(\theta^s\)</span> 为每一层</p>
+<p><span class="math display">\[q = - \nabla _ { \theta } f ( \theta ^ {
+s } )\]</span></p>
+<p><span class="math display">\[r = \nabla _ { \theta } L ( z _ { m } ,
+\theta ^ { s } )\]</span></p>
+<p>拆分到层</p>
+<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx q ^ { T
+} ( \hat { G } + \lambda I ) ^ { - 1 } r = \sum _ { \ell = 1 } ^ { L } q
+_ { \ell } ^ { T } ( \hat { G } _ { \ell } + \lambda I ) ^ { - 1 } r _ {
+\ell }\]</span></p>
+<p>而对于每条训练数据 <span class="math inline">\(z_m\)</span> ，<span
+class="math inline">\(r\)</span> 可以更进一步的拆分为每个 token ( <span
+class="math inline">\(r =\sum_t r_t\)</span> )，所以有</p>
+<p><span class="math display">\[I _ { f } ( z _ { m } ) \approx \sum _ {
+\ell = 1 } ^ { L } \sum _ { t = 1 } ^ { T } q _ { \ell } ^ { T } ( \hat
+{ G } _ { \ell } + \lambda I ) ^ { - 1 } r _ { \ell , t }\]</span></p>
+<p>但是对于 token 层级来说，每个 token
+都是包含整条数据的信息（之前所有输入的 token），所以并不能独立观察。</p>
+<p>额外说明：如果 Predident George Washington 具有影响力，因为预测了
+George，则 President （前一个 token）将会高亮显示。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tracing/0b2bafe670a7427a90200dd60de5aa83.webp" /></p>
 ]]></content>
       <categories>
-        <category>DeepLearning</category>
+        <category>PaperReading</category>
       </categories>
       <tags>
-        <tag>CUDA</tag>
-        <tag>PyTorch</tag>
-        <tag>Cpp</tag>
+        <tag>NLP</tag>
+        <tag>Interpretability</tag>
       </tags>
   </entry>
   <entry>
-    <title>torch中的scheduler</title>
-    <url>/hexo_blog/2021/02/03/PyTorch/torch-scheduler%E8%AE%B0%E5%BD%95/</url>
-    <content><![CDATA[<p>PyTorch学习率调整过程中版本问题引发的不同结果解析</p>
+    <title>ipv6免流</title>
+    <url>/hexo_blog/2020/09/16/ipv6%E5%85%8D%E6%B5%81/</url>
+    <content><![CDATA[<p>借助ssh端口转发，使用ipv6绕过流量计费限制，前提是ipv6不被计费且拥有不被流量限制的ipv6设备。。。</p>
 <span id="more"></span>
-<p>代码</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">optimizer1 = torch.optim.SGD([torch.randn(<span class="number">1</span>, requires_grad=<span class="literal">True</span>)], lr=<span class="number">1e-3</span>)</span><br><span class="line">exp_lr_scheduler1 = torch.optim.lr_scheduler.MultiStepLR(optimizer1,</span><br><span class="line">                                             milestones=[<span class="number">5</span>, <span class="number">10</span>], gamma=<span class="number">0.1</span>)</span><br><span class="line"></span><br><span class="line">optimizer2 = torch.optim.SGD([torch.randn(<span class="number">1</span>, requires_grad=<span class="literal">True</span>)], lr=<span class="number">1e-3</span>)</span><br><span class="line">exp_lr_scheduler2 = torch.optim.lr_scheduler.MultiStepLR(optimizer2,</span><br><span class="line">                                             milestones=[<span class="number">5</span>, <span class="number">10</span>], gamma=<span class="number">0.1</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">15</span>):</span><br><span class="line">    exp_lr_scheduler1.step()</span><br><span class="line">    exp_lr_scheduler2.step(epoch)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;Epoch &#123;&#125;, lr1 &#123;&#125;, lr2 &#123;&#125;&#x27;</span>.<span class="built_in">format</span>(epoch,</span><br><span class="line">        optimizer1.param_groups[<span class="number">0</span>][<span class="string">&#x27;lr&#x27;</span>],</span><br><span class="line">        optimizer2.param_groups[<span class="number">0</span>][<span class="string">&#x27;lr&#x27;</span>]))</span><br></pre></td></tr></table></figure>
-<p>当torch版本为1.2.0时，输出如下：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Epoch <span class="number">1</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">2</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">3</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">4</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">5</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">6</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">7</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">8</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">9</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">10</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">11</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">12</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">13</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">14</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br></pre></td></tr></table></figure>
-<p>当torch版本为1.4.0时，输出如下：</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Epoch <span class="number">1</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">2</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">3</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">4</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">5</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">6</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">7</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">8</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">9</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">10</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">11</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">12</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">13</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">14</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br></pre></td></tr></table></figure>
-<p>小结：<code>scheduler.step()</code>中的
-<code>epoch</code>参数由于版本问题会带来不同的作用效果。1.2.0版本是判断epoch是否在某个区间内，而1.4.0版本是会直接调整学习到达最后一个区间。</p>
+<p>学校校园网一个月使用流量有限制，但发现上传是不计入流量统计的（不过对个人没啥利益，或者利用点比较少。。）。另外发现学校给分配ipv6，然后突发奇想，ipv6是不是没有被计费。。。然后利用家里的树莓派（有ipv6，可直连）测了一下上传下载速度，发现真的没有被计费。。。。</p>
+<p>所以可以直接使用端口转发来达到免流目的，见
+<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>，一行命令即可。</p>
+<p>当然，另一个操作其实是借助v2ray+国外服务器+规则文件+IBM免费的中转，这样搭配使用更加舒适（可以跑满百兆网）。</p>
 ]]></content>
       <categories>
         <category>Note</category>
       </categories>
       <tags>
-        <tag>PyTorch</tag>
-        <tag>scheduler</tag>
+        <tag>Linux</tag>
+        <tag>ipv6</tag>
       </tags>
   </entry>
   <entry>
-    <title>交叉熵的反向传播 Step by Step（PyTorch）</title>
-    <url>/hexo_blog/2023/09/03/PyTorch/%E4%BA%A4%E5%8F%89%E7%86%B5%E7%9A%84%E5%8F%8D%E5%90%91%E4%BC%A0%E6%92%AD%20Step%20by%20Step/</url>
-    <content><![CDATA[<p>本文从公式开始，一步步用PyTorch实现自定义的交叉熵损失函数，最后理解分布式损失函数</p>
+    <title>vLLM  的 PageAttention</title>
+    <url>/hexo_blog/2023/11/24/vLLM%E7%9A%84PageAttention/</url>
+    <content><![CDATA[<p>介绍 vLLM 的 PageAttention 加速</p>
+<p><a href="https://blog.vllm.ai/2023/06/20/vllm.html">博客</a></p>
+<p><a href="https://arxiv.org/pdf/2309.06180.pdf">论文</a></p>
+<p><a
+href="https://github.com/vllm-project/vllm/blob/7c600440f7560348e571f021f2b2d1469de5264d/csrc/attention/attention_kernels.cu#L71-L383">代码</a></p>
 <span id="more"></span>
-<h2 id="step-1a-simple-case">Step 1：A Simple Case</h2>
-<p>以交叉熵为例，普通的二分类问题 <span class="math display">\[
-\text {Binary Cross Entropy} = -\left (y \log (\hat {y}) + (1-y) \log
-(1-\hat {y})\right)
-\]</span> 更常见的多分类问题 <span class="math display">\[
-\text {Multiclass Cross Entropy} = -\sum_{c=0}^{C} y_c \log (\hat {y}_c)
-\]</span> 其中，<span class="math inline">\(y\)</span>
-表示真实的标签值（通常是one-hot向量），<span
-class="math inline">\(\hat{y}\)</span> 表示模型的输出，<span
-class="math inline">\(C\)</span> 表示有多少类别需要分类</p>
-<p>在 PyTorch 中调用（本文使用的torch version==2.0.0）</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">criterion = torch.nn.CrossEntropyLoss()</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])   <span class="comment"># \hat&#123;y&#125; Shape: bs, C</span></span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])                 <span class="comment"># y       Shape: bs</span></span><br><span class="line"></span><br><span class="line">loss = criterion(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398)</span></span><br></pre></td></tr></table></figure>
-<p>这里的<code>outputs</code>是模型的输出，<code>targets</code>是真实的标签值</p>
-<h2 id="step-2-手写">Step 2: 手写</h2>
-<p>实现<code>CrossEntropyLoss</code>需要两个条件</p>
-<ol type="1">
-<li>使用 PyTorch 写出损失函数的公式，以及对应的导数</li>
-<li>根据公式计算loss，并计算梯度（grad）进行反向传播</li>
-</ol>
-<h3 id="公式">公式</h3>
-<p>从<span class="math inline">\(-\sum_{c=0}^{C} y_c \log (\hat
-{y}_c)\)</span>出发</p>
-<ol type="1">
-<li>由于模型输出并不是单纯的概率值，因此会对模型输出进行<code>softmax</code>计算，得到<span
-class="math inline">\(\hat{y}\)</span> <span
-class="math display">\[\text {softmax}(x_i) = \frac
-{e^{x_i}}{\sum_{j=1}^n e^{x_j}}\]</span></li>
-<li>再以简单的单分类任务来说，
-<ol type="1">
-<li><span class="math inline">\(y\)</span> = [1, 0, 0], <span
-class="math inline">\(\hat{y}\)</span> = [0.5, 0.2, 0.3] -&gt; log(<span
-class="math inline">\(\hat{y}\)</span>)=[-0.9398, -1.2398, -1.1398]</li>
-<li>计算时，只有c=0时，结果有值（-0.9398），其他类别时为0，无需计算</li>
-<li>因此，实际最终结果就是取出对应类别的 <span
-class="math inline">\(\hat{y}\)</span> 的log值</li>
-</ol></li>
-</ol>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">MyCrossEntropyLoss</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># outputs.shape: bs, C</span></span><br><span class="line">    <span class="comment"># targets.shape: bs</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  <span class="comment"># 对预测结果进行softmax操作</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 使用索引选择对应类别的概率值，并使用负对数函数计算损失</span></span><br><span class="line">    loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) <span class="comment"># 对每个计算取对应的target</span></span><br><span class="line">    <span class="keyword">return</span> loss.mean() <span class="comment"># 最终返回结果的平均值</span></span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])	<span class="comment"># \hat&#123;y&#125; Shape: bs, C</span></span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])	</span><br><span class="line"></span><br><span class="line">loss = MyCrossEntropyLoss(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398)</span></span><br></pre></td></tr></table></figure>
-<h4 id="梯度">梯度</h4>
-<ol type="1">
-<li><p>结合前面的计算，每次计算loss时，实际上只有对应类别的计算是有效的，因此将其简化为<span
-class="math inline">\(-y_c\log(\hat{y}_c)\)</span>，而由于<span
-class="math inline">\(y_c=1\)</span>，所以其实只有<span
-class="math inline">\(-\log(\hat{y}_c)\)</span></p></li>
-<li><p>这个公式只需要计算log的导数。对于<span
-class="math inline">\(f(x)=\log(x)\)</span>，其导数<span
-class="math inline">\(f^{&#39;}(x)=\frac{1}{x}\)</span>。</p></li>
-<li><p>因此，这里的梯度很简单，如下所示，其中，<span
-class="math inline">\(y_c=1\)</span> <span class="math display">\[
-\frac {\partial \text {CrossEntropy}(y, \hat {y})}{\partial \hat {y}_c}
-= -\frac {y_c}{\hat {y}_c}=-\frac{1}{\hat{y}_c}
-\]</span></p></li>
-<li><p>而对于模型的输出而言，还有一个SoftMax计算。SoftMax的梯度有</p>
-<ol type="1">
-<li>当 <span class="math inline">\(i = j\)</span> 时：</li>
-</ol>
-<p><span class="math display">\[
-\frac {\partial y_i}{\partial x_i} = \frac {e^{x_i}\left (\sum_{k=1}^n
-e^{x_k}\right) - e^{x_i} e^{x_i}}{\left (\sum_{k=1}^n e^{x_k}\right)^2}
-= y_i (1-y_i)
-\]</span></p>
-<ol start="2" type="1">
-<li>当 <span class="math inline">\(i \neq j\)</span> 时：</li>
-</ol>
-<p><span class="math display">\[
-\frac {\partial y_i}{\partial x_j} = -\frac {e^{x_j} e^{x_i}}{\left
-(\sum_{k=1}^n e^{x_k}\right)^2} = -y_i y_j
-\]</span></p></li>
-<li><p>由于求导的链式法则，因此将SoftMax的导数乘以交叉熵的导数，就能得到模型输出的梯度。综上，对于模型的输出，交叉熵的梯度仅在目标标签上有效（≠0），所以最后的梯度计算为，</p></li>
-</ol>
-<p><span class="math display">\[-\frac{1}{\hat{y}_c}\times
-y_c(1-y_c)=y_c-1\]</span></p>
-<h3 id="autograd">autograd</h3>
-<p>在PyTorch内部有一套完善的求导机制。因此，仅需按照其规则进行书写即可实现前向传播、反向传播（链式求导）。如下所示，在forward填写loss的计算，backward中填写grad的计算即可</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">CrossEntropyLoss</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">        <span class="comment"># outputs：模型的输出</span></span><br><span class="line">        <span class="comment"># targets: 真实标签</span></span><br><span class="line">        ...</span><br><span class="line">        ctx.save_for_backward(...) <span class="comment"># 需要在反向传播中使用的变量</span></span><br><span class="line">        <span class="keyword">return</span> loss</span><br><span class="line">    </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">        <span class="comment"># grad_output: 反向传播的梯度</span></span><br><span class="line">        ... = ctx.saved_tensors</span><br><span class="line">        ...</span><br><span class="line">	    <span class="keyword">return</span> new_grad_output</span><br></pre></td></tr></table></figure>
-<h4 id="forward">Forward</h4>
-<p>这里基本就是把公式照抄过来，但需要额外把梯度计算时用到的变量存一下。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  </span><br><span class="line"></span><br><span class="line">    loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) </span><br><span class="line">    ctx.save_for_backward(outputs, targets)</span><br><span class="line">    <span class="keyword">return</span> loss.mean()</span><br></pre></td></tr></table></figure>
-<h4 id="backward">Backward</h4>
-<p>这里的<code>grad_output</code>
-表示梯度计算的输出。它是一个与前向传播函数的输出形状相同的张量。在反向传播过程中，我们将计算当前函数的导数，乘以
-<code>grad_output</code>
-作为输入。这个值代表了后续节点对当前节点的梯度贡献。</p>
-<p>通过将 <code>grad_output</code>
-乘以导数（也称为雅可比向量积），可以有效地传递梯度信息到较早的节点，从而实现自动微分。</p>
-<p>注意，<code>grad_output</code>
-的形状必须与函数的输出形状一致，否则会引发错误。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">    outputs, targets = ctx.saved_tensors</span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    </span><br><span class="line">    grad_y_pred = outputs.clone()</span><br><span class="line">    grad_y_pred[<span class="built_in">range</span>(bs), targets] -= <span class="number">1</span>  <span class="comment"># 计算对应类别的梯度</span></span><br><span class="line">    grad_y_pred /= bs <span class="comment"># 取均值</span></span><br><span class="line">    </span><br><span class="line">    grad_y_pred *= grad_output.item()  <span class="comment"># 乘以关于loss的梯度</span></span><br><span class="line">    </span><br><span class="line">    <span class="keyword">return</span> grad_y_pred, <span class="literal">None</span></span><br></pre></td></tr></table></figure>
-<h3 id="完整版">完整版</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyCrossEntropyLossFunction</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">        bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">        outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  </span><br><span class="line"></span><br><span class="line">        loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) </span><br><span class="line">        ctx.save_for_backward(outputs, targets)</span><br><span class="line">        <span class="keyword">return</span> loss.mean()</span><br><span class="line">    </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">        outputs, targets = ctx.saved_tensors</span><br><span class="line">        bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">        </span><br><span class="line">        grad_y_pred = outputs.clone()</span><br><span class="line">        grad_y_pred[<span class="built_in">range</span>(bs), targets] -= <span class="number">1</span>  <span class="comment"># 只需计算对应标签的梯度</span></span><br><span class="line">        </span><br><span class="line">        grad_y_pred /= bs <span class="comment"># 取均值</span></span><br><span class="line">        </span><br><span class="line">        grad_y_pred *= grad_output.item()  <span class="comment"># 乘以关于loss的梯度</span></span><br><span class="line">        </span><br><span class="line">        <span class="keyword">return</span> grad_y_pred, <span class="literal">None</span></span><br><span class="line">  </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">MyCrossEntropyLoss</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="keyword">return</span> MyCrossEntropyLossFunction.apply(outputs, targets)</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]], requires_grad=<span class="literal">True</span>)</span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])</span><br><span class="line">loss = MyCrossEntropyLoss(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398, grad_fn=&lt;MyCrossEntropyLossFunctionBackward&gt;)</span></span><br></pre></td></tr></table></figure>
-<p>结合简单的双层MLP</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> copy</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MLP</span>(torch.nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="built_in">super</span>().__init__()</span><br><span class="line">        <span class="variable language_">self</span>.l1 = torch.nn.Linear(<span class="number">5</span>, <span class="number">5</span>, bias=<span class="literal">False</span>)</span><br><span class="line">        <span class="variable language_">self</span>.l2 = torch.nn.Linear(<span class="number">5</span>, <span class="number">3</span>, bias=<span class="literal">False</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.l2(<span class="variable language_">self</span>.l1(x))</span><br><span class="line">    </span><br><span class="line">input_ = torch.rand((<span class="number">5</span>, <span class="number">5</span>))</span><br><span class="line">labels = torch.randint(<span class="number">0</span>, <span class="number">3</span>, size=(<span class="number">5</span>,))</span><br><span class="line">model1 = MLP()</span><br><span class="line">model2 = copy.deepcopy(model1)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">10</span>, <span class="string">&quot;Before Train&quot;</span>, <span class="string">&quot;=&quot;</span>*<span class="number">10</span>)</span><br><span class="line"><span class="built_in">print</span>(model1.l1.weight)</span><br><span class="line"><span class="built_in">print</span>(model2.l1.weight)</span><br><span class="line"></span><br><span class="line">criterion1 = torch.nn.CrossEntropyLoss()</span><br><span class="line">criterion2 = MyCrossEntropyLoss</span><br><span class="line"></span><br><span class="line">optim1 = torch.optim.Adam(model1.parameters(), lr=<span class="number">1e-2</span>)</span><br><span class="line">optim2 = torch.optim.Adam(model2.parameters(), lr=<span class="number">1e-2</span>)</span><br><span class="line"></span><br><span class="line">outputs1 = model1(copy.deepcopy(input_))</span><br><span class="line">loss1 = criterion1(outputs1, labels)</span><br><span class="line">optim1.zero_grad()</span><br><span class="line">loss1.backward()</span><br><span class="line">optim1.step()</span><br><span class="line"></span><br><span class="line">outputs2 = model2(copy.deepcopy(input_))</span><br><span class="line">loss2 = criterion2(outputs2, labels)</span><br><span class="line">optim2.zero_grad()</span><br><span class="line">loss2.backward()</span><br><span class="line">optim2.step()</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">10</span>, <span class="string">&quot;After Train&quot;</span>, <span class="string">&quot;=&quot;</span>*<span class="number">10</span>)</span><br><span class="line"><span class="built_in">print</span>(model1.l1.weight)</span><br><span class="line"><span class="built_in">print</span>(model2.l1.weight)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h2 id="step-3-分布式">Step 3: 分布式</h2>
-<p>模型训练速度提升+单卡放不下大模型，由于这两个需求，所以分布式计算中需要重写损失函数的forward和backward，以Megatron-LM的训练代码为例，如下所示。接下来将逐步拆解实现过程（忽略label_smoothing，将其看作0.0）。</p>
-<p><a
-href="https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/cross_entropy.py">cross_entropy.py</a></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> megatron.core.parallel_state <span class="keyword">import</span> (</span><br><span class="line">    get_tensor_model_parallel_group,</span><br><span class="line">    get_tensor_model_parallel_rank,</span><br><span class="line">    get_tensor_model_parallel_world_size,</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> .utils <span class="keyword">import</span> VocabUtility</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">_VocabParallelCrossEntropy</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, vocab_parallel_logits, target, label_smoothing=<span class="number">0.0</span></span>):</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Maximum value along vocab dimension across all GPUs.</span></span><br><span class="line">        logits_max = torch.<span class="built_in">max</span>(vocab_parallel_logits, dim=-<span class="number">1</span>)[<span class="number">0</span>]</span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group()</span><br><span class="line">        )</span><br><span class="line">        <span class="comment"># Subtract the maximum value.</span></span><br><span class="line">        vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Get the partition&#x27;s vocab indecies</span></span><br><span class="line">        get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size</span><br><span class="line">        partition_vocab_size = vocab_parallel_logits.size()[-<span class="number">1</span>]</span><br><span class="line">        rank = get_tensor_model_parallel_rank()</span><br><span class="line">        world_size = get_tensor_model_parallel_world_size()</span><br><span class="line">        vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Create a mask of valid vocab ids (1 means it needs to be masked).</span></span><br><span class="line">        target_mask = (target &lt; vocab_start_index) | (target &gt;= vocab_end_index)</span><br><span class="line">        masked_target = target.clone() - vocab_start_index</span><br><span class="line">        masked_target[target_mask] = <span class="number">0</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Get predicted-logits = logits[target].</span></span><br><span class="line">        <span class="comment"># For Simplicity, we convert logits to a 2-D tensor with size</span></span><br><span class="line">        <span class="comment"># [*, partition-vocab-size] and target to a 1-D tensor of size [*].</span></span><br><span class="line">        logits_2d = vocab_parallel_logits.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line">        masked_target_1d = masked_target.view(-<span class="number">1</span>)</span><br><span class="line">        arange_1d = torch.arange(start=<span class="number">0</span>, end=logits_2d.size()[<span class="number">0</span>], device=logits_2d.device)</span><br><span class="line">        predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]</span><br><span class="line">        predicted_logits_1d = predicted_logits_1d.clone().contiguous()</span><br><span class="line">        predicted_logits = predicted_logits_1d.view_as(target)</span><br><span class="line">        predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line">        <span class="comment"># All reduce is needed to get the chunks from other GPUs.</span></span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            predicted_logits,</span><br><span class="line">            op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">            group=get_tensor_model_parallel_group(),</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Sum of exponential of logits along vocab dimension across all GPUs.</span></span><br><span class="line">        exp_logits = vocab_parallel_logits</span><br><span class="line">        torch.exp(vocab_parallel_logits, out=exp_logits)</span><br><span class="line">        sum_exp_logits = exp_logits.<span class="built_in">sum</span>(dim=-<span class="number">1</span>)</span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            sum_exp_logits,</span><br><span class="line">            op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">            group=get_tensor_model_parallel_group(),</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Loss = log(sum(exp(logits))) - predicted-logit.</span></span><br><span class="line">        loss = torch.log(sum_exp_logits) - predicted_logits</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Normalize and optionally smooth logits</span></span><br><span class="line">        exp_logits.div_(sum_exp_logits.unsqueeze(dim=-<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line">        vocab_size = exp_logits.size(-<span class="number">1</span>)</span><br><span class="line">        <span class="keyword">if</span> label_smoothing &gt; <span class="number">0</span>:</span><br><span class="line">            <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            We&#x27;d like to assign 1 / (K - 1) probability mass to every index that is not the ground truth.</span></span><br><span class="line"><span class="string">            = (1 - alpha) * y_gt + alpha * mean(y_&#123;i for i != gt&#125;)</span></span><br><span class="line"><span class="string">            = (1 - alpha) * y_gt + (alpha / (K - 1)) * \sum_&#123;i != gt&#125; y_i</span></span><br><span class="line"><span class="string">            = ((K - 1) * (1 - alpha) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_&#123;i != gt&#125; y_i</span></span><br><span class="line"><span class="string">            = (K * (1 - alpha) - 1) / (K - 1)) * y_gt  + (alpha / (K - 1)) * \sum_&#123;i&#125; y_i</span></span><br><span class="line"><span class="string">            = (1 - (alpha * K) / (K - 1)) * y_gt + ( (alpha * K) / (K - 1) ) * \sum_&#123;i&#125; y_i / K</span></span><br><span class="line"><span class="string">            From: https://github.com/NVIDIA/NeMo/blob/main/nemo/collections/common/losses/smoothed_cross_entropy.py</span></span><br><span class="line"><span class="string">            &quot;&quot;&quot;</span></span><br><span class="line">            <span class="keyword">assert</span> <span class="number">1.0</span> &gt; label_smoothing &gt; <span class="number">0.0</span></span><br><span class="line">            smoothing = label_smoothing * vocab_size / (vocab_size - <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># Exp logits at this point are normalized probabilities. So we can just take the log to get log-probs.</span></span><br><span class="line">            log_probs = torch.log(exp_logits)</span><br><span class="line">            mean_log_probs = log_probs.mean(dim=-<span class="number">1</span>)</span><br><span class="line">            loss = (<span class="number">1.0</span> - smoothing) * loss - smoothing * mean_log_probs</span><br><span class="line"></span><br><span class="line">        ctx.label_smoothing, ctx.vocab_size = label_smoothing, vocab_size</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Store softmax, target-mask and masked-target for backward pass.</span></span><br><span class="line">        ctx.save_for_backward(exp_logits, target_mask, masked_target_1d)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> loss</span><br><span class="line"></span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Retreive tensors from the forward path.</span></span><br><span class="line">        softmax, target_mask, masked_target_1d = ctx.saved_tensors</span><br><span class="line">        label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size</span><br><span class="line"></span><br><span class="line">        <span class="comment"># All the inputs have softmax as thier gradient.</span></span><br><span class="line">        grad_input = softmax</span><br><span class="line">        <span class="comment"># For simplicity, work with the 2D gradient.</span></span><br><span class="line">        partition_vocab_size = softmax.size()[-<span class="number">1</span>]</span><br><span class="line">        grad_2d = grad_input.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Add the gradient from matching classes.</span></span><br><span class="line">        arange_1d = torch.arange(start=<span class="number">0</span>, end=grad_2d.size()[<span class="number">0</span>], device=grad_2d.device)</span><br><span class="line"></span><br><span class="line">        softmax_update = <span class="number">1.0</span> - target_mask.view(-<span class="number">1</span>).<span class="built_in">float</span>()</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> label_smoothing &gt; <span class="number">0</span>:</span><br><span class="line">            smoothing = label_smoothing * vocab_size / (vocab_size - <span class="number">1</span>)</span><br><span class="line">            grad_2d[arange_1d, masked_target_1d] -= (<span class="number">1.0</span> - smoothing) * softmax_update</span><br><span class="line">            average_grad = <span class="number">1</span> / vocab_size</span><br><span class="line">            grad_2d[arange_1d, :] -= smoothing * average_grad</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            grad_2d[arange_1d, masked_target_1d] -= softmax_update</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Finally elementwise multiplication with the output gradients.</span></span><br><span class="line">        grad_input.mul_(grad_output.unsqueeze(dim=-<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> grad_input, <span class="literal">None</span>, <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">vocab_parallel_cross_entropy</span>(<span class="params">vocab_parallel_logits, target, label_smoothing=<span class="number">0.0</span></span>):</span><br><span class="line">    <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">    Performs cross entropy loss when logits are split across tensor parallel ranks</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Arguments:</span></span><br><span class="line"><span class="string">        vocab_parallel_logits: logits split across tensor parallel ranks</span></span><br><span class="line"><span class="string">                               dimension is [sequence_length, batch_size, hidden_size]</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        target: correct vocab ids of dimseion [sequence_length, micro_batch_size]</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        lobal_smoothing: smoothing factor, must be in range [0.0, 1.0)</span></span><br><span class="line"><span class="string">                         default is no smoothing (=0.0)</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    <span class="keyword">return</span> _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, label_smoothing)</span><br></pre></td></tr></table></figure>
-<h3 id="为什么">为什么</h3>
-<p>本文不介绍分布式训练的其他原理，仅关注模型最后的输出，以及损失函数的计算。</p>
-<p>前文已经详细介绍了交叉熵的实现，那么为什么到分布式训练的时候，就不能直接复用呢？主要原因在于模型的输出被拆散了。回顾模型的输出，其shape为[<code>bs</code>,
-<code>C</code>]，<code>bs</code>表示每次训练的数据量<code>batch size</code>，<code>C</code>表示任务的总的分类数量，在LLM中主要是词表大小<code>vocab_size</code>（下文用此代表类别数量）。</p>
-<p>对于模型的输出而言，分布式（假设有<code>word_size</code>张GPU）主要分两种情况
-1.
-拆分<code>bs</code>，每张卡上有<code>bs</code>数据量，那么整合模型的最后输出应为[<code>bs</code>
-* <code>word_size</code>, <code>vocab_size</code>] 2.
-拆分<code>vocab_size</code>，单个模型的输出为[<code>bs</code>,
-<code>vocab_size</code> //
-<code>word_size</code>]。这种情况需要保证<code>vocab_size</code>能够整除<code>word_size</code></p>
-<p>因此，在进行loss计算前，需要先同步所有的模型输出，然后再进行loss计算。这里的同步，就是从所有GPU中获取模型的输出。但考虑到通信开销问题，有些操作是可以进行优化的，即无需通信全部内容再计算。重新思考损失的计算过程可以分为如下几步：</p>
+<p>文中 GIF 图来源于 vLLM 博客</p>
+<h2 id="动机">动机</h2>
+<h3 id="定义">定义</h3>
+<p>当前模型生成的机理是，给定 Prompt，输出
+Completion。模型在生成时有一个参数
+MAX_SEQ_LEN，以控制模型最大能处理的长度（Prompt + Completion）。</p>
+<h3 id="问题">问题</h3>
+<p>经典的框架在生成时，会提前申请 MAX_SEQ_LEN 的长度。然而，最后生成的
+Prompt + Completion 的长度往往可能是小于 MAX_SEQ_LEN
+的。从而导致有很大显存的浪费，且生成之后，该部分显存又将丢弃，从而导致操作的浪费。</p>
+<h3 id="小结">小结</h3>
+<p>传统方法在生成时，会有申请/丢弃显存的 <strong>IO
+浪费</strong>，模型在处理时会有冗余显存始终未被利用到，导致模型能处理的
+Prompt 低于理想情况，<strong>计算浪费</strong>。</p>
+<h2 id="怎么做">怎么做</h2>
+<p>冗余的部分来自于，attention
+计算中的空间。因此，借鉴操作系统中虚拟内存和分页的经典思想，将空间打碎，从而实现更高效的空间利用。</p>
+<p>翻译后的原句：PagedAttention
+允许在非连续的内存空间中存储连续的键和值。具体来说，PagedAttention
+将每个序列的 KV
+缓存划分为块，每个块包含固定数量的标记的键和值。在注意力计算过程中，PagedAttention
+内核有效地识别并获取这些块。</p>
+<p>kv cache 空间计算：</p>
+<p><span class="math inline">\(2 \times 5120 \times 40 \times 2 \times
+2048 / (1024 ** 3)\)</span> GB = 1.5625 GB</p>
+<p>(k+v) <span class="math inline">\(\times\)</span> hidden_size <span
+class="math inline">\(\times\)</span> num_layers <span
+class="math inline">\(\times\)</span> FP16 bytes <span
+class="math inline">\(\times\)</span> tokens_num</p>
+<p>对于 13B 模型 2048 的 seq_len 而言，需要申请 1.56 GB 的显存。而如果
+Prompt + Completion 的长度只有 1024，那么将浪费 0.78 G
+的显存（在没有其他 kv 优化的情况下）</p>
+<p>如下图所示，block 存储了所有可能的 token。在生成时，模型从 block
+中进行计算。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/vllm_pageattention/annimation0.gif" /></p>
+<p>Attention 计算公式</p>
+<p><span class="math display">\[a _ { i j } = \frac { \operatorname {
+exp } ( q _ { i } ^ { T } k _ { j } / \sqrt { d } ) } { \sum _ { t = 1 }
+^ { i } \operatorname { exp } ( q _ { i } ^ { T } k _ { t } / \sqrt { d
+} ) } , o _ { i } = \sum _ { j = 1 } ^ { i } a _ { i j } v _ { j
+}\]</span></p>
+<p>PageAttention 计算公式</p>
+<p><span class="math display">\[A _ { i j } = \frac { \operatorname {
+exp } ( q _ { i } ^ { T } K _ { j } / \sqrt { d } ) } { \sum _ { t = 1 }
+^ { \lceil i / B \rceil } \operatorname { exp } ( q _ { i } ^ { T } K _
+{ t } 1 / \sqrt { d } ) } , o _ { i } = \sum _ { j = 1 } ^ { \lceil i /
+B \rceil } V _ { j } A _ { i j } ^ { T }\]</span></p>
+<p>对于 query (q)，不需要改变，而 k 和 v 转变为 block
+的形式。计算步骤如下</p>
 <ol type="1">
-<li>SoftMax计算：<span class="math inline">\(\exp(x) /
-\sum(\exp(x)\)</span></li>
-<li>log的计算： <span class="math inline">\(-\log[\exp(x) /
-\sum(\exp(x)]\)</span></li>
-<li>索引：根据targets索引矩阵对应位置 <span class="math inline">\(x[...,
-...]\)</span></li>
+<li>提取对应的 block</li>
+<li>计算 q 和 block 中每个元素的结果，汇总会注意力分数 A</li>
+<li>注意力分数 A 与 block 中 V 相乘，得到输出</li>
 </ol>
-<p>其中，前两者跟最后一个步骤是可以独立运行的。而前两者的计算是耦合的，为节约通信量，对log计算进行变化</p>
+<h2 id="细节实现">细节实现</h2>
+<p>如下图所示，kv cache 分了 logical block 和 physical
+block，前者和后者内容是一样的，通过 Block table
+找到在显存中的实际位置。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/vllm_pageattention/annimation1.gif" /></p>
+<p>而如果是多个句子同时请求，那么在显存足够的情况下，多个句子会同时处理。如下所示，每个句子有自己的
+logical block，共同映射到 physical block 中。（多个句子会一起计算）</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/vllm_pageattention/annimation3.gif" /></p>
+<h2 id="代码实现">代码实现</h2>
+<p>调度是通过 Python
+实现的，这一部分逻辑有些杂乱，之后再看。这里主要关注 PageAttention
+的计算，核心的代码如下</p>
+<h3 id="背景知识">背景知识</h3>
+<p><code>#pragma unroll</code>
+控制了其紧跟着的下一个循环的展开（为了加速使用，下面将删除对应行）</p>
+<p>__syncthreads();
+是CUDA并行编程中的一个同步指令，用于确保在同一个线程块内的所有线程都到达此指令位置后才能继续执行后面的代码，意味着在这个函数之前的所有操作都已经完成，buffer或者内存的数据已经得到同步更新。（下面将删除对应行）</p>
+<p>__shfl_xor_sync：进行数据交换和归约操作</p>
+<p>__shfl_sync：广播操作</p>
+<p><code>WARP_SIZE：</code>在CUDA编程中，<code>WARP_SIZE</code>通常指的是一个warp中的线程数。warp是CUDA硬件的一个基本执行单元，每个warp包含了一定数量的线程，这些线程会同时执行相同的指令。<code>WARP_SIZE 通常定义为</code>32，这意味着每个warp包含32个线程。</p>
+<p>逐步拆建，首先，确定这段函数在干啥。它代替了 Python
+的以下计算，简单来说有两个矩阵乘法，以及一个 softmax
+操作。涉及到的变量有 q、k、v。（忽略 mask）</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">score = torch.matmul(q, k.transpose(<span class="number">2</span>, <span class="number">3</span>)) / math.sqrt(<span class="variable language_">self</span>.head_dim)</span><br><span class="line">score = nn.functional.softmax(score, dim=-<span class="number">1</span>, dtype=torch.float32).to(q.dtype)</span><br><span class="line">out = torch.matmul(score, v)</span><br></pre></td></tr></table></figure>
+<p>之后，再捋一捋变量的数据结构，从下面的代码开始，假设 q、k、v 的
+num_head 和 head_size 是相同的</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Copy from https://www.dipkumar.dev/becoming-the-unbeatable/posts/gpt-kvcache/</span></span><br><span class="line"> </span><br><span class="line"><span class="comment"># 一行 Attention 的版本</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">attention</span>(<span class="params">q, k, v, mask</span>):</span><br><span class="line">    <span class="keyword">return</span> softmax(q @ k.T / np.sqrt(q.shape[-<span class="number">1</span>]) + mask) @ v</span><br><span class="line">   </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func</span>(<span class="params">x, kvcache</span>):</span><br><span class="line">    <span class="comment"># n 条句子，每个句子有 n_embd 的向量，分别经过 qkv 的 mlp，可以得到每个句子的 q、k、v</span></span><br><span class="line">    <span class="comment"># 其维度分别为 [n_seq, n_embd]</span></span><br><span class="line">    x = mlp(x, **c_attn)  <span class="comment"># [n_seq, n_embd] -&gt; [n_seq, 3*n_embd]</span></span><br><span class="line">    qkv = np.split(x, <span class="number">3</span>, axis=-<span class="number">1</span>)  <span class="comment"># [n_seq, 3*n_embd] -&gt; [3, n_seq, n_embd]</span></span><br><span class="line"> </span><br><span class="line">    <span class="comment"># 在第 i-1 个 token 的生成过程中，为了加速会利用之前生成 token 的 kv_cache。为什么？因为要在之前的 token 中查询</span></span><br><span class="line">    <span class="comment"># 所以，实际上 k 和 v 的结构为 [prev_n_seq+n_seq, n_embd]</span></span><br><span class="line">    <span class="keyword">if</span> kvcache:</span><br><span class="line">        new_q, new_k, new_v = qkv</span><br><span class="line">        old_k, old_v = kvcache</span><br><span class="line">        k = np.vstack([old_k, new_k])</span><br><span class="line">        v = np.vstack([old_v, new_v])</span><br><span class="line">        qkv = [new_q, k, v]</span><br><span class="line"> </span><br><span class="line">    current_cache = [qkv[<span class="number">1</span>], qkv[<span class="number">2</span>]]</span><br><span class="line"> </span><br><span class="line">    <span class="comment"># split into heads</span></span><br><span class="line">    <span class="comment"># 这里又需要根据 head 来切分，所以结构更新为</span></span><br><span class="line">    <span class="comment"># q: [n_head, n_seq,            n_embd/n_head]</span></span><br><span class="line">    <span class="comment"># k: [n_head, prev_n_seq+n_seq, n_embd/n_head]</span></span><br><span class="line">    <span class="comment"># v: [n_head, prev_n_seq+n_seq, n_embd/n_head]</span></span><br><span class="line">    qkv_heads = <span class="built_in">list</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: np.split(x, n_head, axis=-<span class="number">1</span>), qkv))  <span class="comment"># [3, n_seq, n_embd] -&gt; [n_head, 3, n_seq, n_embd/n_head]</span></span><br><span class="line"> </span><br><span class="line">    <span class="comment"># causal mask to hide future inputs from being attended to</span></span><br><span class="line">    <span class="keyword">if</span> kvcache:</span><br><span class="line">        causal_mask = np.zeros((<span class="number">1</span>, k.shape[<span class="number">0</span>]))</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        causal_mask = (<span class="number">1</span> - np.tri(x.shape[<span class="number">0</span>])) * -<span class="number">1e10</span>  <span class="comment"># [n_seq, n_seq]</span></span><br><span class="line"> </span><br><span class="line">    <span class="comment"># perform attention over each head</span></span><br><span class="line">    out_heads = [attention(q, k, v, causal_mask) <span class="keyword">for</span> q, k, v <span class="keyword">in</span> <span class="built_in">zip</span>(*qkv_heads)]  <span class="comment"># [n_head, 3, n_seq, n_embd/n_head] -&gt; [n_head, n_seq, n_embd/n_head]</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<p>而在 PageAttention 中，k 和 v 分别用 block
+再拆了一遍，所以需要多两个维度 num_blocks 以及
+block_size。并且，为了实现连续 Batch，n_seq 将会被拼成一个 seq
+处理。所以最后结构如下：</p>
+<ul>
+<li>q: [n_seq, n_head, head_size]</li>
+<li>k: [num_blocks, n_head, head_size, block_size]</li>
+<li>v: [num_blocks, n_head, head_size, block_size]</li>
+</ul>
+<p>这里的 head_size = n_embd/n_head，且调换了一些维度。</p>
+<p>而对于输出 out: [n_seq, n_head, head_size]</p>
+<p>接下来看关键实现。假设有两个请求，seq1 = [1, 2, 3], seq2 = [1, 3, 4,
+5]。vLLM 会将其拼成一个 seq = [1,2,3,1,3,4,5]</p>
+<p>取 q</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 这里的传参 q // [num_seqs, num_heads, head_size]</span></span><br><span class="line"><span class="comment">// 首先是转成了指针的形式，其次是用了线程，以方便 CUDA 进行加速</span></span><br><span class="line"> </span><br><span class="line"><span class="comment">// seq_idx 表示第 idx 条句子，q_stride 表示这条句子的长度</span></span><br><span class="line"><span class="comment">// head_idx 表示第 idx 个head，HEAD_SIZE 是 head 的大小</span></span><br><span class="line">  <span class="type">const</span> <span class="type">scalar_t</span>* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// THREAD_GROUP_SIZE：线程组的大小；NUM_VECS_PER_THREAD：表示每个线程处理的向量数量（并行）</span></span><br><span class="line">  __shared__ Q_vec q_vecs[THREAD_GROUP_SIZE][NUM_VECS_PER_THREAD];</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = thread_group_idx; i &lt; NUM_VECS_PER_THREAD; i += NUM_THREAD_GROUPS) &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;</span><br><span class="line">    q_vecs[thread_group_offset][i] = *<span class="built_in">reinterpret_cast</span>&lt;<span class="type">const</span> Q_vec*&gt;(q_ptr + vec_idx * VEC_SIZE);</span><br><span class="line">  &#125;</span><br><span class="line"><span class="comment">// 使用 q_vecs 来存储</span></span><br></pre></td></tr></table></figure>
+<p>取 k 和 第一个矩阵乘法</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 这里的传参是 k_cache  const scalar_t* __restrict__ k_cache,   // [num_blocks, num_kv_heads, head_size/x, block_size, x]</span></span><br><span class="line"><span class="comment">// 这里在拿到每组 k_vecs 后，直接计算了点乘（并且考虑了 mask）</span></span><br><span class="line"><span class="comment">// 输出存储在 logits</span></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> WARP_SIZE 32</span></span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line">  <span class="type">float</span> qk_max = -FLT_MAX;</span><br><span class="line"><span class="comment">// 处理每个 block table 内的 key</span></span><br><span class="line">  <span class="type">const</span> <span class="type">int</span>* block_table = block_tables + seq_idx * max_num_blocks_per_seq;</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> block_idx = start_block_idx + warp_idx; block_idx &lt; end_block_idx; block_idx += NUM_WARPS) &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int64_t</span> physical_block_number = <span class="built_in">static_cast</span>&lt;<span class="type">int64_t</span>&gt;(block_table[block_idx]);</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 为了多线程加速，将其拆分为多个块同时处理。可以理解为这里是并行？</span></span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_TOKENS_PER_THREAD_GROUP; i++) &#123;</span><br><span class="line">      <span class="type">const</span> <span class="type">int</span> physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;</span><br><span class="line">      <span class="type">const</span> <span class="type">int</span> token_idx = block_idx * BLOCK_SIZE + physical_block_offset;</span><br><span class="line">      K_vec k_vecs[NUM_VECS_PER_THREAD];</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 获取到 block 中 所有的 k 值</span></span><br><span class="line">      <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; NUM_VECS_PER_THREAD; j++) &#123;</span><br><span class="line">        <span class="type">const</span> <span class="type">scalar_t</span>* k_ptr = k_cache + physical_block_number * kv_block_stride</span><br><span class="line">                                        + kv_head_idx * kv_head_stride</span><br><span class="line">                                        + physical_block_offset * x;</span><br><span class="line">        <span class="type">const</span> <span class="type">int</span> vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;</span><br><span class="line">        <span class="type">const</span> <span class="type">int</span> offset1 = (vec_idx * VEC_SIZE) / x;</span><br><span class="line">        <span class="type">const</span> <span class="type">int</span> offset2 = (vec_idx * VEC_SIZE) % x;</span><br><span class="line">        k_vecs[j] = *<span class="built_in">reinterpret_cast</span>&lt;<span class="type">const</span> K_vec*&gt;(k_ptr + offset1 * BLOCK_SIZE * x + offset2);</span><br><span class="line">      &#125;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 计算点积</span></span><br><span class="line">      <span class="type">float</span> qk = scale * Qk_dot&lt;<span class="type">scalar_t</span>, THREAD_GROUP_SIZE&gt;::<span class="built_in">dot</span>(q_vecs[thread_group_offset], k_vecs);</span><br><span class="line">      qk += (alibi_slope != <span class="number">0</span>) ? alibi_slope * (token_idx - context_len + <span class="number">1</span>) : <span class="number">0</span>;</span><br><span class="line"> </span><br><span class="line">      <span class="keyword">if</span> (thread_group_offset == <span class="number">0</span>) &#123;</span><br><span class="line"><span class="comment">// 考虑 mask</span></span><br><span class="line">        <span class="type">const</span> <span class="type">bool</span> mask = token_idx &gt;= context_len;</span><br><span class="line">        logits[token_idx - start_token_idx] = mask ? <span class="number">0.f</span> : qk;</span><br><span class="line"><span class="comment">// 获取最大值，以计算 softmax</span></span><br><span class="line">        qk_max = mask ? qk_max : <span class="built_in">fmaxf</span>(qk_max, qk);</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br></pre></td></tr></table></figure>
+<p>计算 softmax</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 获取所有 warp 中的最大值</span></span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> mask = WARP_SIZE / <span class="number">2</span>; mask &gt;= THREAD_GROUP_SIZE; mask /= <span class="number">2</span>) &#123;</span><br><span class="line">    qk_max = <span class="built_in">fmaxf</span>(qk_max, __shfl_xor_sync(<span class="built_in">uint32_t</span>(<span class="number">-1</span>), qk_max, mask));</span><br><span class="line">  &#125;</span><br><span class="line"><span class="comment">// 放到共享存储中，只需要第一个线程放置即可</span></span><br><span class="line">  <span class="keyword">if</span> (lane == <span class="number">0</span>) &#123;</span><br><span class="line">    red_smem[warp_idx] = qk_max;</span><br><span class="line">  &#125;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 获取所有 block 中的最大值</span></span><br><span class="line">  qk_max = lane &lt; NUM_WARPS ? red_smem[lane] : -FLT_MAX;</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> mask = NUM_WARPS / <span class="number">2</span>; mask &gt;= <span class="number">1</span>; mask /= <span class="number">2</span>) &#123;</span><br><span class="line">    qk_max = <span class="built_in">fmaxf</span>(qk_max, __shfl_xor_sync(<span class="built_in">uint32_t</span>(<span class="number">-1</span>), qk_max, mask));</span><br><span class="line">  &#125;</span><br><span class="line"><span class="comment">// 将最大值广播至所有线程</span></span><br><span class="line">  qk_max = __shfl_sync(<span class="built_in">uint32_t</span>(<span class="number">-1</span>), qk_max, <span class="number">0</span>);</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 计算每一项的 exp，exp_sum 是分母，logits[i] 是分子</span></span><br><span class="line">  <span class="type">float</span> exp_sum = <span class="number">0.f</span>;</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = thread_idx; i &lt; num_tokens; i += NUM_THREADS) &#123;</span><br><span class="line">    <span class="type">float</span> val = __expf(logits[i] - qk_max);</span><br><span class="line">    logits[i] = val;</span><br><span class="line">    exp_sum += val;</span><br><span class="line">  &#125;</span><br><span class="line">  exp_sum = <span class="built_in">block_sum</span>&lt;NUM_WARPS&gt;(&amp;red_smem[NUM_WARPS], exp_sum);</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 用 1 / 分母，便于直接乘上该值，得到 softmax 后的结果</span></span><br><span class="line">  <span class="type">const</span> <span class="type">float</span> inv_sum = __fdividef(<span class="number">1.f</span>, exp_sum + <span class="number">1e-6f</span>);</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = thread_idx; i &lt; num_tokens; i += NUM_THREADS) &#123;</span><br><span class="line">    logits[i] *= inv_sum;</span><br><span class="line">  &#125;</span><br></pre></td></tr></table></figure>
+<p>取 v 和第二个矩阵乘法</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 假设为 BLOCK_SIZE</span></span><br><span class="line">  <span class="keyword">constexpr</span> <span class="type">int</span> V_VEC_SIZE = <span class="built_in">MIN</span>(<span class="number">16</span> / <span class="built_in">sizeof</span>(<span class="type">scalar_t</span>), BLOCK_SIZE);</span><br><span class="line">  <span class="keyword">using</span> V_vec = <span class="keyword">typename</span> Vec&lt;<span class="type">scalar_t</span>, V_VEC_SIZE&gt;::Type;</span><br><span class="line">  <span class="keyword">using</span> L_vec = <span class="keyword">typename</span> Vec&lt;<span class="type">scalar_t</span>, V_VEC_SIZE&gt;::Type;</span><br><span class="line">  <span class="keyword">using</span> Float_L_vec = <span class="keyword">typename</span> FloatVec&lt;L_vec&gt;::Type;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 所以这里可以为 1</span></span><br><span class="line">  <span class="keyword">constexpr</span> <span class="type">int</span> NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;</span><br><span class="line"><span class="comment">// 这里就是 WARP_SIZE</span></span><br><span class="line">  <span class="keyword">constexpr</span> <span class="type">int</span> NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;</span><br><span class="line"><span class="comment">// 拆分 HEAD_SIZE 到每个线程上</span></span><br><span class="line">  <span class="keyword">constexpr</span> <span class="type">int</span> NUM_ROWS_PER_THREAD = <span class="built_in">DIVIDE_ROUND_UP</span>(HEAD_SIZE, NUM_ROWS_PER_ITER);</span><br><span class="line"> </span><br><span class="line"><span class="comment">// accs 是最后计算的结果，会赋值给返回变量。</span></span><br><span class="line"><span class="comment">// 这里用一维实现，那么其大小应该 &gt;= n_seq * n_head * head_size</span></span><br><span class="line"><span class="comment">// 为什么可以大于？因为有 block，允许一定的冗余</span></span><br><span class="line"><span class="comment">// 为什么是一维？n_seq 被整合成了一个，n_head * head_size 被拆分到了各个线程中（简单理解为每个线程中都有这么一个数组，所以实际是两个维度），最后需要拆汇总？</span></span><br><span class="line">  <span class="type">float</span> accs[NUM_ROWS_PER_THREAD];</span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_ROWS_PER_THREAD; i++) &#123;</span><br><span class="line">    accs[i] = <span class="number">0.f</span>;</span><br><span class="line">  &#125;</span><br><span class="line"> </span><br><span class="line">  <span class="type">scalar_t</span> zero_value;</span><br><span class="line">  <span class="built_in">zero</span>(zero_value);</span><br><span class="line"><span class="comment">// 同上，取每个 block 中的 v 值，以及取出 q*k 的结果 logits</span></span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> block_idx = start_block_idx + warp_idx; block_idx &lt; end_block_idx; block_idx += NUM_WARPS) &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int64_t</span> physical_block_number = <span class="built_in">static_cast</span>&lt;<span class="type">int64_t</span>&gt;(block_table[block_idx]);</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> token_idx = block_idx * BLOCK_SIZE + physical_block_offset;</span><br><span class="line">    L_vec logits_vec;</span><br><span class="line"><span class="comment">// 取出每个句子的 logits</span></span><br><span class="line">    <span class="built_in">from_float</span>(logits_vec, *<span class="built_in">reinterpret_cast</span>&lt;Float_L_vec*&gt;(logits + token_idx - start_token_idx));</span><br><span class="line"><span class="comment">// 拿到 v 的指针，根据指针取值</span></span><br><span class="line">    <span class="type">const</span> <span class="type">scalar_t</span>* v_ptr = v_cache + physical_block_number * kv_block_stride</span><br><span class="line">                                    + kv_head_idx * kv_head_stride;</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_ROWS_PER_THREAD; i++) &#123;</span><br><span class="line">      <span class="type">const</span> <span class="type">int</span> row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;</span><br><span class="line">      <span class="keyword">if</span> (row_idx &lt; HEAD_SIZE) &#123;</span><br><span class="line">        <span class="type">const</span> <span class="type">int</span> offset = row_idx * BLOCK_SIZE + physical_block_offset;</span><br><span class="line"><span class="comment">// 拿到 v 的值</span></span><br><span class="line">        V_vec v_vec = *<span class="built_in">reinterpret_cast</span>&lt;<span class="type">const</span> V_vec*&gt;(v_ptr + offset);</span><br><span class="line">        <span class="keyword">if</span> (block_idx == num_context_blocks - <span class="number">1</span>) &#123;</span><br><span class="line">          <span class="type">scalar_t</span>* v_vec_ptr = <span class="built_in">reinterpret_cast</span>&lt;<span class="type">scalar_t</span>*&gt;(&amp;v_vec);</span><br><span class="line">          <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; V_VEC_SIZE; j++) &#123;</span><br><span class="line"><span class="comment">// 避免 nan 值，置 0 处理</span></span><br><span class="line">            v_vec_ptr[j] = token_idx + j &lt; context_len ? v_vec_ptr[j] : zero_value;</span><br><span class="line">          &#125;</span><br><span class="line">        &#125;</span><br><span class="line"><span class="comment">// 计算第二个矩阵乘法</span></span><br><span class="line">        accs[i] += <span class="built_in">dot</span>(logits_vec, v_vec);</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 每个线程进行求和，汇总至 accs</span></span><br><span class="line">  <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_ROWS_PER_THREAD; i++) &#123;</span><br><span class="line">    <span class="type">float</span> acc = accs[i];</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> mask = NUM_V_VECS_PER_ROW / <span class="number">2</span>; mask &gt;= <span class="number">1</span>; mask /= <span class="number">2</span>) &#123;</span><br><span class="line">      acc += __shfl_xor_sync(<span class="built_in">uint32_t</span>(<span class="number">-1</span>), acc, mask);</span><br><span class="line">    &#125;</span><br><span class="line">    accs[i] = acc;</span><br><span class="line">  &#125;</span><br><span class="line"> </span><br><span class="line">... # 中间一些处理逻辑</span><br><span class="line"> </span><br><span class="line"><span class="comment">// 将 accs 的结果，还原至返回 out 中（out_ptr 是它的指针）</span></span><br><span class="line"><span class="meta">#<span class="keyword">pragma</span> unroll</span></span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; NUM_ROWS_PER_THREAD; i++) &#123;</span><br><span class="line">      <span class="type">const</span> <span class="type">int</span> row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;</span><br><span class="line">      <span class="keyword">if</span> (row_idx &lt; HEAD_SIZE &amp;&amp; lane % NUM_V_VECS_PER_ROW == <span class="number">0</span>) &#123;</span><br><span class="line">        <span class="built_in">from_float</span>(*(out_ptr + row_idx), accs[i]);</span><br><span class="line">      &#125;</span><br><span class="line">    &#125;</span><br><span class="line">  &#125;</span><br></pre></td></tr></table></figure>
+<h2 id="next-step">Next Step</h2>
+<p>对于目前的 LLM 来说，一种广泛应用的场景会固定若干个 System
+Message，相当于 Prompt 是重复的。但每次生成，都需要重新计算每个 System
+Message 的 kv cache。因此，这一步是可以优化的。vLLM
+在论文中提到了这一点，但目前 vLLM 尚未实现。</p>
+<h3 id="system-message-预生成-kv-cache">System Message 预生成 kv
+cache</h3>
+<p>可以先给定若干 System Message，计算好 kv
+cache，以字典的形式存储。在请求时，带上标识符，在字典中找到对应的 kv
+cache，在生成时直接拼上，而不计算这一部分的 kv cache。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/vllm_pageattention/annimation2.gif" /></p>
+<h3 id="策略赋值更新-kv-cache">策略赋值更新 kv cache</h3>
+<p>以工具调用为例，可以预先生成工具调用结果的所有 token 的 kv
+cache，加速生成。</p>
 <ol type="1">
-<li><span class="math inline">\(-\log[\exp(x) /
-\sum(\exp(x)]\)</span></li>
-<li><span class="math inline">\(-\log[\exp(x)] +
-\log[\sum\exp(x)]\)</span></li>
-<li><span class="math inline">\(\log[\sum\exp(x)] - x\)</span></li>
+<li>在实现 multi token attention 的基础上</li>
+<li>调度处理</li>
 </ol>
-<p>因此，可以由原始的<code>func1</code>推导至分布式的<code>func2</code>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func1</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># 原始版本</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    <span class="comment"># Step 0:</span></span><br><span class="line">    predicted_logits = outputs[<span class="built_in">range</span>(bs), targets]</span><br><span class="line"></span><br><span class="line">    predicted_logits = torch.softmax(predicted_logits, dim=<span class="number">1</span>)</span><br><span class="line">    loss = -torch.log(predicted_logits)</span><br><span class="line">    <span class="built_in">print</span>(loss)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func2</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># 分布式的简化版本</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">    min_c, max_c = <span class="number">0</span>, <span class="number">3</span> <span class="comment"># 预先设定的最小和最大值</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 只选择在该设备上的数值</span></span><br><span class="line">    target_mask = (targets &lt; min_c) | (targets &gt;= max_c)</span><br><span class="line">    <span class="comment"># 由于在分布式中min_c不一定为0，所以需要做一个归一化</span></span><br><span class="line">    masked_target = targets.clone() - min_c</span><br><span class="line">    masked_target[target_mask] = min_c</span><br><span class="line"></span><br><span class="line">    <span class="comment"># arange_1d, masked_target 分别对应 上文的range(bs), targets</span></span><br><span class="line">    arange_1d = torch.arange(start=<span class="number">0</span>, end=bs)</span><br><span class="line">    predicted_logits = outputs[arange_1d, masked_target]</span><br><span class="line">    </span><br><span class="line">    predicted_logits = predicted_logits.clone().contiguous()</span><br><span class="line">    <span class="comment"># 对于非该设备上计算的值，置为0。则不进行计算</span></span><br><span class="line">    predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line">    <span class="comment"># 先算第二项</span></span><br><span class="line">    predicted_logits = torch.<span class="built_in">sum</span>(predicted_logits, dim=-<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">    loss = torch.log(torch.exp(outputs).<span class="built_in">sum</span>(dim=-<span class="number">1</span>)) - predicted_logits</span><br><span class="line">    <span class="built_in">print</span>(loss)</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])</span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])    </span><br><span class="line"></span><br><span class="line">func1(outputs, targets)</span><br><span class="line">func2(outputs, targets)</span><br></pre></td></tr></table></figure></p>
-<h3 id="forward-1">Forward</h3>
-<h4 id="预处理">预处理</h4>
-<p>一般而言，为了数值稳定性，SoftMax计算前会减去最大值，即</p>
-<p><span class="math display">\[\text {softmax}(x_i) = \frac
-{e^{x_i}}{\sum_{j=1}^n e^{x_j}} = \frac {e^{x_i - \max(x)}}{\sum_{j=1}^n
-e^{x_j - \max(x)}}\]</span></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 每张卡上的模型输出为 vocab_parallel_logits，shape：[bs, seq_len, vocab_size // word_size]</span></span><br><span class="line"><span class="comment"># SoftMax的前置计算，可以无需通信全部参数，分两步进行</span></span><br><span class="line"><span class="comment"># 1. 计算每张卡的最大值</span></span><br><span class="line"><span class="comment"># 2. 再计算所有卡的最大值</span></span><br><span class="line"><span class="comment"># 3. 最后对每卡张减去最大值</span></span><br><span class="line"><span class="comment"># 这样就能避免通信所有的参数</span></span><br><span class="line">logits_max = torch.<span class="built_in">max</span>(vocab_parallel_logits, dim=-<span class="number">1</span>)[<span class="number">0</span>]</span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group()</span><br><span class="line">)</span><br><span class="line"><span class="comment"># Subtract the maximum value.</span></span><br><span class="line">vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
-<p>这里的<code>seq_len</code>是LLM中句子的长度，句子的每个位置（token）会计算下一个token的概率（在词表中选），因此，可以理解为每个位置都是一个分类任务，因此<code>vocab_size</code>就是分类的数量。即，可以转换为[<code>bs</code>*<code>seq_len</code>,
-<code>vocab_size</code> // <code>word_size</code>]的形式</p>
-<p>由于单卡不一定存在所有词表的值，因此需要在这张卡上将非该卡输出位置的值置为0。
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Get the partition&#x27;s vocab indecies</span></span><br><span class="line">get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size</span><br><span class="line">partition_vocab_size = vocab_parallel_logits.size()[-<span class="number">1</span>]</span><br><span class="line">rank = get_tensor_model_parallel_rank()</span><br><span class="line">world_size = get_tensor_model_parallel_world_size()</span><br><span class="line">vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Create a mask of valid vocab ids (1 means it needs to be masked).</span></span><br><span class="line">target_mask = (target &lt; vocab_start_index) | (target &gt;= vocab_end_index)</span><br><span class="line">masked_target = target.clone() - vocab_start_index</span><br><span class="line">masked_target[target_mask] = <span class="number">0</span></span><br></pre></td></tr></table></figure></p>
-<h4 id="根据-targets-选择模型真实输出的logits">根据 targets
-选择模型真实输出的Logits</h4>
-<p>将模型的输出和标签 转换为[<code>bs</code>*<code>seq_len</code>,
-<code>vocab_size</code> // <code>word_size</code>]的形式，以便于计算
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">logits_2d = vocab_parallel_logits.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line">masked_target_1d = masked_target.view(-<span class="number">1</span>) </span><br><span class="line"></span><br><span class="line"><span class="comment"># 选择真实标签的logits，相当于原来的outputs[range(bs), targets]</span></span><br><span class="line"><span class="comment"># 相当于把此步骤置前，而对于后续计算无影响，并减少了后续的通信时间</span></span><br><span class="line">arange_1d = torch.arange(start=<span class="number">0</span>, end=logits_2d.size()[<span class="number">0</span>], device=logits_2d.device)</span><br><span class="line">predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]</span><br><span class="line">predicted_logits_1d = predicted_logits_1d.clone().contiguous()</span><br></pre></td></tr></table></figure></p>
-<h4 id="计算loss">计算loss</h4>
-<p>将logits转换为原来的shape，再求和，先得到了最后的第一项，之后再减去log
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">predicted_logits = predicted_logits_1d.view_as(target)</span><br><span class="line">predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line"><span class="comment"># All reduce is needed to get the chunks from other GPUs.</span></span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    predicted_logits,</span><br><span class="line">    op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">    group=get_tensor_model_parallel_group(),</span><br><span class="line">)</span><br></pre></td></tr></table></figure></p>
-<p>计算SoftMax <span class="math display">\[\text {softmax}(x_i) = \frac
-{e^{x_i}}{\sum_{j=1}^n e^{x_j}}\]</span></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">exp_logits = vocab_parallel_logits</span><br><span class="line"><span class="comment"># 计算每张卡的exp，作为分子</span></span><br><span class="line">torch.exp(vocab_parallel_logits, out=exp_logits)</span><br><span class="line"><span class="comment"># 再计算这张卡上的exp之和</span></span><br><span class="line">sum_exp_logits = exp_logits.<span class="built_in">sum</span>(dim=-<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 再计算所有卡的exp之和，作为分母</span></span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    sum_exp_logits,</span><br><span class="line">    op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">    group=get_tensor_model_parallel_group(),</span><br><span class="line">)</span><br></pre></td></tr></table></figure>
-<p>计算loss <figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 第一项是log(sum(exp(logits)))，第二项是最后的logits</span></span><br><span class="line">loss = torch.log(sum_exp_logits) - predicted_logits</span><br><span class="line"></span><br><span class="line"><span class="comment"># 为了Backward，需要保存一些变量</span></span><br><span class="line">exp_logits.div_(sum_exp_logits.unsqueeze(dim=-<span class="number">1</span>))</span><br></pre></td></tr></table></figure></p>
-<h3 id="backward-1">Backward</h3>
-<h4 id="预处理-1">预处理</h4>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">softmax, target_mask, masked_target_1d = ctx.saved_tensors</span><br><span class="line">label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据SoftMax计算的结果，忽略无需计算梯度的位置。target_mask中的元素不是0就是1</span></span><br><span class="line">grad_input = softmax</span><br><span class="line"><span class="comment"># For simplicity, work with the 2D gradient.</span></span><br><span class="line">partition_vocab_size = softmax.size()[-<span class="number">1</span>]</span><br><span class="line">grad_2d = grad_input.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Add the gradient from matching classes.</span></span><br><span class="line">arange_1d = torch.arange(start=<span class="number">0</span>, end=grad_2d.size()[<span class="number">0</span>], device=grad_2d.device)</span><br><span class="line"></span><br><span class="line">softmax_update = <span class="number">1.0</span> - target_mask.view(-<span class="number">1</span>).<span class="built_in">float</span>()</span><br></pre></td></tr></table></figure>
-<h4 id="梯度计算">梯度计算</h4>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 正常计算位置，就-1，否则将不减（即不做计算）</span></span><br><span class="line">grad_2d[arange_1d, masked_target_1d] -= softmax_update</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最后乘上关于loss的梯度</span></span><br><span class="line">grad_input.mul_(grad_output.unsqueeze(dim=-<span class="number">1</span>))</span><br></pre></td></tr></table></figure>
+<p>对于这两种都依赖于 multi token attention，而这一个 vLLM 是直接调用
+xformers 的 xops.memory_efficient_attention_forward
+实现的。这种方式当前只能在第一次生成时使用，没法拼接？</p>
+<ul>
+<li>比如总共 10 个 prompt token，现在一次性生成 10 个 token 的 kv
+cache。期望分两次各生成前 5 个和后 5 个的 token，将 kv cache
+拼起来，得到的结果不一致</li>
+</ul>
+<p>相关文章：<a href="#">Post not found: PyTorch/pytorch_cpp_extension.md</a></p>
+<p>xops.memory_efficient_attention_forward 学习</p>
 ]]></content>
       <categories>
-        <category>Algorithm</category>
+        <category>Note</category>
       </categories>
       <tags>
-        <tag>PyTorch</tag>
-        <tag>数学</tag>
-        <tag>交叉熵</tag>
+        <tag>NLP</tag>
+        <tag>Attention</tag>
+        <tag>vLLM</tag>
       </tags>
   </entry>
   <entry>
-    <title>OpenWrt路由器折腾记录</title>
-    <url>/hexo_blog/2023/01/10/IOT/OpenWrt%E8%B7%AF%E7%94%B1%E5%99%A8%E8%AE%B0%E5%BD%95/</url>
-    <content><![CDATA[<p>在路由器上安装和配置OpenWrt。它包括安装软件包和设置防火墙、接口和各种网络连接的分步说明。</p>
+    <title>产品评论数据情感分析：从文本评论预处理到模型构建</title>
+    <url>/hexo_blog/2018/08/18/%E4%BA%A7%E5%93%81%E8%AF%84%E8%AE%BA%E6%95%B0%E6%8D%AE%E6%83%85%E6%84%9F%E5%88%86%E6%9E%90/</url>
+    <content><![CDATA[<p>NLP处理问题的框图（知识点）。To Be Continue！</p>
 <span id="more"></span>
-<h2 id="基本界面">基本界面</h2>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673353557758.png" /></p>
-<p>准备：</p>
-<ul>
-<li>固件</li>
-<li>待恢复的配置</li>
-<li>需要安装的软件+运行的命令</li>
-</ul>
-<p>配置出问题，直接重刷。同时方便起见，要及时备份当前的配置。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673353799884.png" /></p>
-<h2 id="软件包的安装">软件包的安装</h2>
-<p>能在线安装的可以直接在线安装。直接从"系统-&gt;软件包"里面找到需要安装的软件。</p>
-<p>但有时候在线安装会由于网络，软件源、软件版本等问题，无法自动安装。因此需要我们手动找到ipk文件进行安装。使用下面的命令进行安装。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">opkg install XXX</span><br></pre></td></tr></table></figure>
-<p>官方软件下载网页：<a
-href="https://downloads.openwrt.org/releases/">https://downloads.openwrt.org/releases/</a></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 系统版本可以直接从管理界面的概况中得知，这里是17.01.4</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 内核版本</span></span><br><span class="line">&gt;&gt;&gt; <span class="built_in">uname</span> -a </span><br><span class="line">Linux dw33d 4.4.194 <span class="comment">#0 Tue Oct 17 17:46:20 2017 mips GNU/Linux</span></span><br><span class="line"><span class="comment"># 查看支持的架构</span></span><br><span class="line">&gt;&gt;&gt; opkg print-architecture</span><br><span class="line"><span class="built_in">arch</span> all 1</span><br><span class="line"><span class="built_in">arch</span> noarch 1</span><br><span class="line"><span class="built_in">arch</span> mips_24kc 10</span><br></pre></td></tr></table></figure>
-<p>对应我这里的第三方包就是：<a
-href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/packages/">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/packages/</a></p>
-<p>基础的库：<a
-href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/base">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/base</a></p>
-<p>luci：<a
-href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/luci/">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/luci/</a></p>
-<h3 id="tmux">tmux</h3>
-<p>以tmux安装为例，该软件无法直接安装。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 1. 从package里面下载tmux的ipkg文件</span></span><br><span class="line"><span class="comment"># 2. 执行安装命令</span></span><br><span class="line">&gt;&gt;&gt;opkg install tmux_2.3-1_mips_24kc.ipk</span><br><span class="line"><span class="comment"># 3. 报错信息，一些依赖文件没有。所以从base里面找到对应缺失的ipk文件。并进行安装</span></span><br><span class="line">&gt;&gt;&gt;opkg install terminfo_6.0-1_mips_24kc.ipk libncurses_6.0-1_mips_24kc.ipk</span><br><span class="line"><span class="comment"># 4. 此时，再安装tmux。安装成功</span></span><br><span class="line">&gt;&gt;&gt;opkg install tmux_2.3-1_mips_24kc.ipk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 但安装libevent2这个库时，提示当前已安装，并且版本过高，并不会自动降级。所以需要手动卸载原有的软件</span></span><br><span class="line">&gt;&gt;&gt;opkg remove libevent2 --force-depends</span><br><span class="line"><span class="comment"># 卸载失败，提示已有软件依赖该库。所以我们需要强制进行卸载，将依赖软件也进行卸载。加上--force 命令</span></span><br><span class="line"><span class="comment"># 之后再安装低版本的libncurses</span></span><br><span class="line">&gt;&gt;&gt;opkg install libevent2_2.0.22-1_mips_24kc.ipk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最后再运行tmux，成功</span></span><br></pre></td></tr></table></figure>
-<p>其他软件应该同理，需要一步一步找到依赖文件，并且耐心地解决各种问题。dw33d内置了一个16g的sd卡，所以我把一些重装必备的ipk文件放到里面，便于快速安装。并且刷写固件并不会影响该sd卡的内容，可以放心存储。</p>
-<h3 id="tailscale">Tailscale</h3>
-<p>异地组网工具，从官网下载找到mips版本，放到sd卡中，避免刷新固件有影响。相当于单文件就可以运行。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压文件</span></span><br><span class="line">&gt;&gt;&gt; tar zxvf tailscale_1.34.1_mips.tgz</span><br><span class="line"><span class="comment"># 到对应的目录中，里面会有一些文件。主要是两个可执行文件tailscale和tailscaled。</span></span><br><span class="line"><span class="comment"># 第一次运行需要启动tailscaled服务</span></span><br><span class="line">&gt;&gt;&gt;./tailscaled --state=tailscaled.state</span><br><span class="line"><span class="comment"># 此时需要新开一个窗口，登录tailscale</span></span><br><span class="line">&gt;&gt;&gt;./tailscale login </span><br><span class="line"><span class="comment"># 可以直接把网址复制出来，用其他设备的浏览器进行登录</span></span><br></pre></td></tr></table></figure>
-<p>登录完毕之后，组网就完成了。此时，可以关闭该窗口。但为了后台启动，所以可以使用tmux，或者nohup命令。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">nohup</span> ./tailscaled --state=tailscaled.state &gt; t.log 2&gt;&amp;1 &amp;</span><br></pre></td></tr></table></figure>
-<p>同时为了保证开机自启动，所以我们可以在web管理界面中"系统-&gt;启动项"中，最后的"本地启动脚本"中插入(<code>exit 0</code>之前行)</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 先切换到对应目录，再执行</span></span><br><span class="line"><span class="built_in">cd</span> /mnt/sda1/tailscale_1.34.1_mips &amp;&amp; <span class="built_in">nohup</span> ./tailscaled --state=tailscaled.state &gt; t.log 2&gt;&amp;1 &amp;</span><br></pre></td></tr></table></figure>
-<p>这样，就能保证异地管理该路由器的web界面，并且可以直接ssh登录</p>
-<h2 id="防火墙">防火墙</h2>
-<p>"网络-&gt;防火墙"中的默认配置，wan口是拒绝入站数据的。这样就会导致如果该路由器跟其他设备同属于一个局域网，ssh和web界面均不能通过ip登录。具体的例子是：</p>
-<ul>
-<li>该路由器连接了一个上级路由器A，获取到的IP是192.168.0.X。分配给下面的设备的IP是192.168.1.X</li>
-<li>另一个设备接入路由器A时，获取到的IP是192.168.0.Y。此时能ping通192.168.0.X，但是无法进行ssh，也无法通过192.168.0.X访问管理界面</li>
-<li>另一个设备接入该路由器时，获取到的IP是192.168.1.Y。此时能够通过ssh和web访问192.168.1.X。</li>
-</ul>
-<p>需求：在同局域网下的其他设备也能访问该路由器。</p>
-<p>做法：</p>
-<ul>
-<li>一个简单的做法就是如下图所示，把"防火墙"中的"常规设置"里面的wan口，入站数据从<strong>拒绝</strong>改成<strong>接受</strong>。（默认是拒绝入站的）</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1680509223.png" /></p>
-<ul>
-<li>另一个做法是配置"防火墙-&gt;端口转发"，把需要配置的端口转发出来。如下图所示，web界面的端口是80。所以从wan口出来的访问80端口的流量，直接转发到192.168.4.1（该路由器分配给下面的IP，不是获取的IP）</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673356610477.png" /></p>
-<p>关于流量规则这里，则是在系统内部决定是否开放某个端口。可以参考默认的一些配置进行开放，一般应该是不需要调整的。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673356804612.png" /></p>
-<h2 id="接口">接口</h2>
-<p>"网络-&gt;接口"</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357070459.png" /></p>
-<p>我主要会有以下四种上网方式（不会专业术语，简单描述现象）：</p>
-<ul>
-<li><p>桥接光猫</p>
-<ul>
-<li>将路由器WAN口接入光猫LAN口，代替光猫进行拨号，需要知道宽带的账号密码。</li>
-<li>在WAN口中，协议选择<strong>PPPoE</strong>，输入账号密码</li>
-<li>此时该路由器就是主路由器，一般同时有ipv4和ipv6两个地址，假设ipv4的ip为192.168.1.1</li>
-<li>接入该路由器的设备，由该路由器进行分配ip，ip与路由器同网段，如192.168.1.2</li>
-</ul></li>
-<li><p>桥接上一级路由器</p>
-<ul>
-<li>将路由器WAN口接入上一级路由器LAN口，无需知道账号密码</li>
-<li>在WAN口中，协议选择<strong>DHCP客户端</strong></li>
-<li>在物理设置中，选择桥接接口，默认<strong>以太网交换机:
-eth0</strong>开启。勾选两个无线网络。如下图所示</li>
-<li><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673524264094.png" /></li>
-<li>此时，IP同上一级路由器同网段，且如果有IPv6，则连接该路由器的设备也有IPv6</li>
-</ul></li>
-<li><p>做NAT</p>
-<ul>
-<li>将路由器WAN接入上一级路由器的LAN口，相当于把该路由器做一个新的入口。</li>
-<li>在WAN中，协议选择<strong>DHCP客户端</strong>或<strong>静态地址</strong>（需要额外配置）</li>
-<li>此时，接入该路由器的设备会独立上一级路由器的设备，即网段不同。</li>
-<li>注：如果此时还需要ipv6，可以参考这篇文件进行配置。<a
-href="https://www.lategege.com/?p=676">https://www.lategege.com/?p=676</a>
-<ul>
-<li>简单来说，就是需要选择静态IP，并在<strong>WAN口</strong>和<strong>LAN</strong>中的"DHCP服务器"中的IPv6设置中，路由器广告服务，DHCPv6服务，NDP-代理三个均选择中继模式</li>
-<li>并且在终端中 <code>vim /etc/config/dhcp</code>，找到
-<code>config dhcp 'wan'</code>对应的选项，加入
-<code>option master '1'</code>这一行</li>
-</ul></li>
-</ul></li>
-<li><p>WiFi放大器</p>
-<ul>
-<li><p>无需网线</p></li>
-<li><p>"网络-&gt;无线"中的搜索，找到需要放大WiFi，加入网络。看情况选择放大2.4G还是5G的WiFi</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357519480.png" /></p></li>
-<li><p>WPA密钥就是WiFi密码，新的网络名称可以默认也可以另起一个以便区分。</p></li>
-<li><p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357576678.png" /></p></li>
-<li><p>此时接口中会多一个接口</p></li>
-<li><p>其他同桥接</p></li>
-</ul></li>
-</ul>
-]]></content>
-      <categories>
-        <category>Linux</category>
-      </categories>
-      <tags>
-        <tag>Linux</tag>
-        <tag>路由器</tag>
-        <tag>OpenWrt</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>安卓手机利用Termux安装Linux</title>
-    <url>/hexo_blog/2021/07/21/IOT/%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA%E5%88%A9%E7%94%A8Termux%E5%AE%89%E8%A3%85Linux/</url>
-    <content><![CDATA[<p>本文介绍如何在安卓手机上安装和运行Ubuntu系统，并通过ssh连接进行操作，同时给出了一些其他有用的命令和配置说明。</p>
-<span id="more"></span>
-<h3 id="安装">安装</h3>
-<p>不要从Google Play上安装，正确的安装包地址<a
-href="https://f-droid.org/packages/com.termux/">f-droid</a></p>
-<p>下载完成后，正常安装，但需要额外的给存储空间权限。</p>
-<p>虽然说可以免root安装，但没有root权限，有些操作不能很好的完成，如
-<code>top</code></p>
-<h3 id="连接">连接</h3>
-<p>在手机上配置太麻烦，所以可以使用同一局域网下的电脑进行配置</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 手机上</span></span><br><span class="line"><span class="comment"># 安装ssh服务</span></span><br><span class="line">pkg install openssh -y</span><br><span class="line"><span class="comment"># 开启ssh服务器</span></span><br><span class="line">sshd</span><br><span class="line"><span class="comment"># 配置密码</span></span><br><span class="line">passwd</span><br><span class="line"><span class="comment"># 输入想要配置的密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 电脑终端, ip填写对应的手机ip，8022是termux默认的ssh端口</span></span><br><span class="line">ssh root@[ip] -p8022</span><br><span class="line"><span class="comment"># 输入刚刚的密码</span></span><br></pre></td></tr></table></figure>
-<p>至此，就能在电脑端进行配置</p>
-<h3 id="配置">配置</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 安装第三方库</span></span><br><span class="line">pkg install proot git python wget -y</span><br><span class="line"></span><br><span class="line"><span class="comment"># 一键安装项目</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/sqlsec/termux-install-linux.git</span><br><span class="line"><span class="built_in">cd</span> termux-install-linux</span><br><span class="line">python termux-linux-install.py</span><br><span class="line"><span class="comment"># 选择Ubuntu</span></span><br><span class="line"><span class="built_in">cd</span> ~/Termux-Linux/Ubuntu</span><br><span class="line">./start-ubuntu.sh</span><br><span class="line"></span><br><span class="line"><span class="comment"># 或者，ubuntu版本，最新支持20.04</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/MFDGaming/ubuntu-in-termux</span><br><span class="line"><span class="built_in">cd</span> ubuntu-in-termux</span><br><span class="line"><span class="built_in">chmod</span> +x ubuntu.sh</span><br><span class="line">./ubuntu.sh -y</span><br><span class="line">./startubuntu.sh</span><br></pre></td></tr></table></figure>
-<p>这样安装的Linux是与Termux共享文件目录的，所以可以直接将文件拷贝在
-<code>ubuntu-fs/root</code>文件夹中，启动
-<code>./startubuntu.sh</code>后，就能直接看到文件。</p>
-<p>注：若进入系统后，无法更新软件源，可能是dns存在问题，可以在termux中更改dns解析（因为系统中不好编辑）。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim ubuntu-fs/etc/resolv.conf</span><br><span class="line"></span><br><span class="line">`</span><br><span class="line">nameserver 114.114.114.114</span><br><span class="line">nameserver 8.8.8.8</span><br><span class="line">`</span><br><span class="line"><span class="comment"># 若依旧不行，可尝试其他dns，如1.1.1.1等</span></span><br></pre></td></tr></table></figure>
-<h3 id="其他">其他</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">pkg install vim tmux -y</span><br><span class="line"></span><br><span class="line"><span class="comment"># frp项目地址，手机请注意下载arm版本，一般为arm64，不能直接在Linux系统中配置，必须得在Termux中操作</span></span><br><span class="line">[https://github.com/fatedier/frp/releases](https://github.com/fatedier/frp/releases)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机温度，需要除以1000</span></span><br><span class="line"><span class="built_in">cat</span> /sys/class/thermal/thermal_zone0/temp</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机电池容量，需root</span></span><br><span class="line"><span class="built_in">cat</span> /sys/class/power_supply/battery/capacity</span><br><span class="line"></span><br><span class="line"><span class="comment"># 配置termux的启动操作</span></span><br><span class="line">vim ~/.bashrc</span><br><span class="line"></span><br><span class="line">`</span><br><span class="line">sshd <span class="comment"># 开启ssh服务，便于连接</span></span><br><span class="line"><span class="comment"># 配置frp相关操作，便于直接远程连接</span></span><br><span class="line"><span class="comment"># 配置进入Linux系统操作</span></span><br><span class="line">`</span><br></pre></td></tr></table></figure>
+<h1 id="产品评论数据情感分析">产品评论数据情感分析</h1>
+<h2 id="数据获取">数据获取</h2>
+<h2 id="文本评论预处理">文本评论预处理</h2>
+<h4 id="编码问题可选">编码问题（可选）</h4>
+<h4 id="文本去重">文本去重</h4>
+<h6 id="基本解释">基本解释</h6>
+<p>大量重复且无实际意义的数据，需要删除</p>
+<h6 id="算法">算法</h6>
+<p>编辑距离去重
+计算两条语料的编辑距离，与阈值（提前设定好的一个值）进行判断，小于阈值就进行去重
+优点：有效去除了接近重复或完全重复的数据
+缺点：错删语义相近的语料，两句话虽然相似但并不需要删除。由于阈值设定的原因可能会错删。
+Simhash算法去重 k-shingle算法(用的不多) Minhash算法</p>
+<h6 id="选用的方法">选用的方法</h6>
+<p>最简单粗暴：完全重复的语料进行剔除。见 <code>programmer_2</code></p>
+<h4 id="机械压缩去词">机械压缩去词</h4>
+<h6 id="基本解释-1">基本解释</h6>
+<p>经过上面的处理之后，还有这样一部分数据并没有去除。如"好的好的好的好的好的好的好的好的好的"#
+我们称之为连续重复语料，最常见的较长的无意义语料。</p>
+<h6 id="处理方法">处理方法</h6>
+<p>在某些情况下，重复词带有一定的情感倾向，如果直接进行处理，会影响情感倾向的判断。
+所以这里需要对语料进行机械压缩去词处理，去除一些连续累赘的表达（不影响情感）。比如上面的句子，压缩为"好的"
+一般情况下，这种词汇常出现在开头或者结尾。语料中间的重复词汇可能具有实际意义比如"滔滔不绝"中的滔滔是具有实际意义的。因此只对开头与结尾的连续重复词进行操作。</p>
+<h6 id="设定规则">设定规则</h6>
+<p>考虑因素：词法结构、文字表达特点 七个规则</p>
+<h4 id="短句删除">短句删除</h4>
+<h6
+id="字数越少表达的意思就少所以需要过滤掉过短的评论文本数据">字数越少表达的意思就少，所以需要过滤掉过短的评论文本数据。</h6>
+<h2 id="文本评论分词">文本评论分词</h2>
+<h4 id="分词介绍">分词介绍</h4>
+<h6 id="python库jieba">python库：jieba</h6>
+<h6 id="引入停用词">引入停用词</h6>
+<h6 id="特征处理">特征处理</h6>
+<h6 id="中文分词方法">中文分词方法</h6>
+<p>基于规则的分词方法 基于统计的分词方法 基于语义的分词方法
+基于理解的分词方法</p>
+<h2 id="模型构建">模型构建</h2>
+<h4 id="情感倾向性模型">情感倾向性模型</h4>
+<h6 id="训练生成词向量">训练生成词向量</h6>
+<p>One-hot# Representation Distributed# Representation word2vec</p>
+<h6 id="人工标注与映射">人工标注与映射</h6>
+<p>snownlp</p>
+<h6 id="训练模型">训练模型</h6>
+<p>栈式自编码网络 基本介绍</p>
+<h4 id="语义网络分析">语义网络分析</h4>
+<h6 id="语义网络">语义网络</h6>
+<p>介绍 步骤 优势</p>
+<h4 id="模型选择">模型选择</h4>
+<h6 id="lda模型的主题分析">LDA模型的主题分析</h6>
+<p>模型介绍 Bag Of Words（BOW，词袋模型） 模型估计 实现过程
+拓展：ILDA</p>
+<h6 id="vsm与tf-idf">VSM与TF-IDF</h6>
+<h6 id="svd与lsa">SVD与LSA</h6>
+<h6 id="plsa与lda">PLSA与LDA</h6>
+<h4 id="构建模型">构建模型</h4>
+<h6 id="svm-rbf-pca">SVM (RBF) + PCA</h6>
+<h6 id="subtopic">Subtopic</h6>
+<h2 id="模型评价">模型评价</h2>
+<h4 id="应用层次分析法">应用层次分析法</h4>
+<h4 id="模糊综合评价">模糊综合评价</h4>
 ]]></content>
       <categories>
-        <category>Android</category>
+        <category>NLP</category>
       </categories>
       <tags>
-        <tag>Linux</tag>
-        <tag>Android</tag>
-        <tag>Termux</tag>
+        <tag>数据分析</tag>
       </tags>
   </entry>
   <entry>
-    <title>对各种电子产品的拓展玩法与互联</title>
-    <url>/hexo_blog/2019/09/14/IOT/%E5%AF%B9%E5%90%84%E7%A7%8D%E7%94%B5%E5%AD%90%E8%AE%BE%E5%A4%87%E7%9A%84%E6%8B%93%E5%B1%95%E7%8E%A9%E6%B3%95%E4%B8%8E%E4%BA%92%E8%81%94/</url>
-    <content><![CDATA[<p>介绍手中各种电子产品及各自的拓展玩法</p>
+    <title>年度推荐软件Tailscale：连接团队设备和开发环境的利器</title>
+    <url>/hexo_blog/2023/04/03/%E5%B9%B4%E5%BA%A6%E6%8E%A8%E8%8D%90%E8%BD%AF%E4%BB%B6Tailscale/</url>
+    <content><![CDATA[<p>本文介绍了 Tailscale
+这款软件可以轻松架设异地组网，解决调试难题、实现流量转发和开放服务等多个需求。</p>
 <span id="more"></span>
-<h2
-id="对各种电子产品的拓展玩法与互联">对各种电子产品的拓展玩法与互联</h2>
-<p>手上有的电子产品，其中大部分应该是每个人都有的。</p>
-<ul>
-<li><p>魅族手机</p></li>
-<li><p>笔记本电脑（Win10）</p></li>
-<li><p>1T的机械移动硬盘</p></li>
-<li><p>若干U盘。。</p></li>
-<li><p>Kindle Paper White 3</p></li>
-<li><p>树莓派Zero</p></li>
-<li><p>IPad 2018（IOS13前）</p></li>
-</ul>
-<h3 id="手机与ipad">手机与IPad</h3>
-<p>这两个放在一起的原因是，在IOS13之前，IPad其实差不多可以等同于大尺寸IPone。所以这里就是一个安卓手机（未ROOT）与IPone的介绍，这里的玩法也算是比较多的，下面一一介绍。关于APP的安装包在此就不提供了，请读者自行搜索。当然还有ADB这种玩法，也不做介绍，可参考
-<a href="#">Post not found: IOT/用ADB调试安卓手机.md</a></p>
-<h4 id="termux与juicessh">Termux与JuiceSSH</h4>
-<p>放在搜索引擎一搜，其实这两个APP就有很多相关介绍了。但这里我想介绍一些不一样的玩法。关于Termux操作不方便这一点，可以使用JuiceSSH连接localhost来操作，具体操作不介绍。</p>
-<p><strong>下载</strong>：如何在手机上实现多线程下载？下一个ADM？当然这样很方便，但我们同样可以使用Termux来完成这一点。Termux安装Axel（<code>pkg install axel</code>），下载<code>axel -n 32 url</code>，使用32个线程下载链接。建议先<code>cd /sdcard/</code>中。</p>
-<p><strong>数据传输</strong>：一个手机和一个IPad，如何在不借助其他介质的前提下快速传输。QQ、微信、快牙等传输文件工具都行，但这里我自己实测速度都不如这里的方法快。就拿快牙举例，它实际是用手机开放一个<code>2.4G</code>的热点，然后用二维码进行链接传输。同样的，我们用同样的思路，手机开放<code>5G</code>热点，IPad手动连接此WIFI。打开Termux，先<code>ifconfig</code>知道本地IP，形如<code>192.168.x.1</code>这种，再<code>cd</code>到传输文件的目录，利用Python的静态资源服务器<code>python3 -m http.server</code>，这样，资源就对局域网内设备（连接到手机热点的设备）开放。接下来操作IPad，由于IOS13之前，<code>Safari</code>不支持输入链接下载，所以这里推荐<code>Alook</code>与<code>Documents</code>（后者免费），在<code>Documents</code>中的浏览器输入刚刚的手机<code>ip:8000</code>，这样就能看到手机资源所在文件目录的所有文件，点击即可下载。实测单个文件速度大概比<code>2.4G</code>快3倍（无需流量）。当然这里手机和IPad也可以同时连接同一个WIFI，然后使用同上方法进行传输（速度取决于路由器和网络带宽）。</p>
-<p><strong>远程手机</strong>：如何在手机不连接WIFI，不连接数据线的前提下，用其他设备访问手机（手机与其他设备的实际距离也可以很远），前提手机能上网。这一方法个人暂没有特别用途，但<code>Just for fun</code>。首先，需要一个公网IP服务器；接下来需要会玩ssh，<code>Termux</code>与服务器配置好ssh，<code>Termux</code>能够ssh到服务器上即可。最后一步，参考
-<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>。</p>
-<h4 id="spacedesk">spacedesk</h4>
-<p>电脑投屏，使用USB数据线连接设备和电脑，用这个软件进行投屏。我觉得挺鸡肋的。。。安卓手机屏幕又太小，测试IPad的时候发现闪退。</p>
-<h4
-id="androidhttpcapture和httpcanary">AndroidHttpCapture和HttpCanary</h4>
-<p>安卓抓包工具，后者的功能比前者更强大一些。有时候身边没有电脑，需要手机抓包看数据，所以就只能直接用手机看数据包了。前者基本满足了大部分场景的抓包数据。但当某些时候（比如某些APP的数据交互不是使用HTTP等常用协议，可能就没办法抓包，此时电脑上没办法直接抓包），所以就需要使用<code>HttpCanary</code>（需要ROOT）来对网络进行代理，在解析数据。测试如果电脑上使用<code>WireShark</code>，确实是可以抓到包的，但是会乱码。所以可能使用<code>HttpCanary</code>还更为方便一些。</p>
-<h4 id="netx与net-analyzer">NetX与Net Analyzer</h4>
-<p>安卓局域网设备分析工具，用的不多，具体不介绍，有兴趣自行下载、</p>
-<h4 id="ios同类软件免费">IOS同类软件，免费</h4>
-<ul>
-<li><p><code>iTerminal</code>，同<code>JuiceSSH</code></p></li>
-<li><p><code>Stream</code>，同<code>AndroidHttpCapture</code></p></li>
-<li><p><code>Fing</code>，同<code>NetX</code></p></li>
-</ul>
-<h4 id="scrcpy">scrcpy</h4>
-<p>在电脑上使用，安卓投屏工具，<a
-href="https://github.com/Genymobile/scrcpy">scrcpy</a>，由于魅族手机一些独特的操作，所以需要替换jar文件，<a
-href="https://github.com/taie02/taie02.github.io/tree/master/2019/03/03/scrcpy_usage">此项目</a>。</p>
-<h4 id="otg">OTG</h4>
-<p>关于OTG这一项，其实是属于外接设备这一项，通过充电接口转USB接口可以接大部分usb设备，比如u盘这种。甚至还可以反向充电，接鼠标键盘等。</p>
-<h3 id="u盘与移动硬盘">U盘与移动硬盘</h3>
-<p>二者都属于移动存储介质，其实也没什么好介绍的。。。可以提几点，Windows
-To
-Go（WTG）可以搞（高性能的移动盘），自己没尝试过。在移动硬盘上装了两个Linux系统。</p>
-<p>关于如何装系统就不介绍了，这里只有一点需要注意的是备份好数据再分区，在装系统的时候引导装载移动硬盘上，要不然移动硬盘上只能在装系统的设备上启动。</p>
-<h3 id="kindle">Kindle</h3>
-<p>这里不介绍越狱之类的操作，只有常规操作。。。</p>
+<p><a href="https://tailscale.com/">Tailscale</a> connects your team's
+devices and development environments for easy access to remote
+resources. （摘自官网）</p>
+<p>于笔者而言，Tailscale 能够用一个软件轻松的解决三大需求。</p>
 <ul>
-<li><p><a
-href="https://bookfere.com/post/19.html">搭建个人推送网站</a></p></li>
-<li><p><a href="https://volmoe.com/">漫画</a></p></li>
-<li><p>Chrome插件：Push to Kindle</p></li>
+<li>异地组网：软件主打功能。就是将物理位置不同的机器通过该软件，能够使它们处于同一网络环境下，方便远程调试。</li>
+<li>流量转发：A机器在A地，可以借助B机器在B地的网络进行上网。</li>
+<li>开放服务：A机器自建Web服务，不仅可以在组网环境下使用，甚至官方提供了公网的链接（https），即可以不在组网环境下也能够访问。</li>
 </ul>
-<h3 id="树莓派">树莓派</h3>
-<p>吃灰派。。。大概也就能跑跑小demo，爬数据，利用空闲网络带宽下东西。同样也可以利用ssh，进行端口转发，远程连接。</p>
-<p>后续再更。。。</p>
+<p>写在开头，免费版的权益如下图所示，支持20个设备，且域名不可自定义（子域名可以），速度在100-300k/s左右。在Android和iOS设备上不支持同时开启两个
+VPN，即 Clash 无法和 Tailscale 同时使用。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680514288836.png" /></p>
+<h2 id="安装">安装</h2>
+<p>首先需要在官网进行注册，值得一提的是注册方式最好选用能够在日常网络环境下能打开的账号。比如选用Github注册的话，之后所有的设备都需要使用Github登录才能组网。其目前无法在一个账号上绑定多种登录方式。</p>
+<p>官网提供了详尽的<a
+href="https://tailscale.com/download">下载</a>安装说明，移动端可以直接下载对应的App，个人电脑和服务器可以使用常规的命令和软件包的方式进行下载安装。故对于一般的机器不做介绍，本节主要介绍路由器、树莓派、群晖的安装方法。其他机型，可以查看这个<a
+href="https://pkgs.tailscale.com/stable/">地址</a>。</p>
+<p>对于Linux设备，安装步骤主要分两步：</p>
+<ol type="1">
+<li>启动服务，<code>sudo tailscale up</code></li>
+<li>加入组网，<code>tailscale ip -4</code></li>
+</ol>
+<p>对于第二步，输入完成后会有一个链接需要使用对应账号进行登录，可以将链接复制到其他设备用浏览器打开登录。在非常规机器上，首先需要确认机器的CPU架构，命令
+<code>uname -a</code>。</p>
+<h3 id="群晖">群晖</h3>
+<p>群晖的<a
+href="https://pkgs.tailscale.com/stable/#spks">下载地址</a>，需要根据系统版本以及CPU架构选择下载。下载完成后，打开群晖的<strong>套件中心</strong>，右上角的<strong>手动安装</strong>，上传安装包，按照步骤完成安装即可。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/170356.png" /></p>
+<p>安装完成后，点击对应的图标，安装对应的步骤即可启动。</p>
+<h3 id="路由器">路由器</h3>
+<p>路由器的<a
+href="https://pkgs.tailscale.com/stable/#static">下载地址</a>，根据CPU架构选择下载。并将安装包上传至路由器，根据对应的路由器安装命令进行解压安装。</p>
+<h3 id="树莓派">树莓派</h3>
+<p>树莓派的<a
+href="https://pkgs.tailscale.com/stable/#raspbian-stretch">下载地址</a>，根据型号三选一下载。解压安装。树莓派Zero的是Raspbian
+Stretch</p>
+<h2 id="异地组网">异地组网</h2>
+<p>当设备安装完成，并登录后，该设备会自动加入组网。可以在<a
+href="https://login.tailscale.com/admin/machines">设备管理</a>界面查看所有的设备，如下图所示。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/171953.png" /></p>
+<p>第一列的设备名称可以根据自己的喜好进行更改，并且可以直接作为域名使用。假设有个机器叫做AAA且用户名为root，那么可以直接
+<code>ssh root@AAA</code>进行登录。或者群晖名为AAA，那么浏览器直接打开AAA就能访问。</p>
+<p>登录的设备默认是若干天（最长180）后需要重新登录一次，可以在最后一列三个点进行编辑，对该功能进行禁用。禁用成功后，会在第一列显示
+<code>Expiry disabled</code></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/172831.png" /></p>
+<h2 id="流量转发">流量转发</h2>
+<p>本节参考<a href="https://tailscale.com/kb/1019/subnets/">文档</a></p>
+<p>正如开头的图显示的，该功能仅支持以一个设备作为流量出口。当有一个局域网的所有设备不方便接入组网时，可以通过接入该局域网的一个设备（如路由器），再访问该局域网的其他设备。如下图，<code>Subnet Router</code>相当于桥梁，承接了组网和本地组网。组网内的设备可以通过Router访问局域网内的设备，反之亦可。接下来在
+<code>Router</code>设备上操作。</p>
+<p><img src="https://tailscale.com/kb/1019/subnets/subnets.png" /></p>
+<h3 id="step-1">Step 1</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 开启端口转发。有的机器可能默认开启，无需操作。</span></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&#x27;net.ipv4.ip_forward = 1&#x27;</span> | <span class="built_in">sudo</span> <span class="built_in">tee</span> -a /etc/sysctl.d/99-tailscale.conf</span><br><span class="line"><span class="built_in">echo</span> <span class="string">&#x27;net.ipv6.conf.all.forwarding = 1&#x27;</span> | <span class="built_in">sudo</span> <span class="built_in">tee</span> -a /etc/sysctl.d/99-tailscale.conf</span><br><span class="line"><span class="built_in">sudo</span> sysctl -p /etc/sysctl.d/99-tailscale.conf</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建Subnet，192.168.x.y可以换成10.0这种。可以只写一个，x.y替换任意数字</span></span><br><span class="line"><span class="built_in">sudo</span> tailscale up --advertise-routes=192.168.x.y/24,192.168.x.z/24</span><br></pre></td></tr></table></figure>
+<h3 id="step-2">Step 2</h3>
+<p>在机器管理界面，点击对应机器的右侧三个点，再点击
+<code>Edit route settings</code>。如下图所示，两个功能均开启。若仅需实现设备访问，不需要流量转发，则可以不开启第二个。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680522033220.png" /></p>
+<h3 id="step-3">Step 3</h3>
+<p>自此，完成了桥接局域网和组网的功能。要实现流量转发，在Tailscale中叫做Exit
+Node。一方面需要在上一步开启第二个功能，此时Router就具备了作为流量出口的能力。另一方面，假设此时组网内的A设备需要借助Router的流量出口，就需要在A设备上进行操作。</p>
+<p>在Router设备上，</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建Subnet，192.168.x.y可以换成10.0这种。可以只写一个，x.y替换任意数字</span></span><br><span class="line"><span class="built_in">sudo</span> tailscale up --advertise-exit-node --advertise-routes=192.168.x.y/24</span><br></pre></td></tr></table></figure>
+<p>一般有图形化界面的机器，可以直接在软件层面完成。如Windows，此时系统的所有流量均为走这个接口。</p>
+<p><strong>Windows</strong> <img
+src="https://tailscale.com/kb/1103/exit-nodes/exit-node-windows-menu.png" /></p>
+<p><strong>Linux</strong></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">sudo</span> tailscale up --exit-node=&lt;exit-node-ip&gt;</span><br></pre></td></tr></table></figure>
+<p>该功能主要的用途有以下几点</p>
+<ul>
+<li>本节开头所提到的，通过一个Router，从而无需所有设备参与链接。</li>
+<li>流量转发，如果有一个节点在海外。那么可以以此来做跳板</li>
+<li>本文开头所提到的，移动设备上无法与其他VPN并存，那么此时通过这种方式，在Router设备上做分流。</li>
+</ul>
+<h2 id="开放服务">开放服务</h2>
+<p>截至本文完成日期，该功能还在Beta阶段，并且要求版本&gt;=1.38.3。<a
+href="https://tailscale.com/kb/1223/tailscale-funnel/">详细文档</a></p>
+<h3 id="step-1-1">Step 1</h3>
+<p>如果仅需在组网内访问，则可以跳过该步骤。</p>
+<p>在<a href="https://login.tailscale.com/admin/acls">Access
+controls</a>控制页面加入配置，新增 <code>nodeAttrs</code>和修改
+<code>groups</code>，其中
+<code>wnma3mz@github</code>替换为对应账号。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680523192944.png" />
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/tailscale/1680523204260.png" /></p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="attr">&quot;groups&quot;</span><span class="punctuation">:</span> <span class="punctuation">&#123;</span></span><br><span class="line">     <span class="attr">&quot;group:can-funnel&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">     <span class="string">&quot;wnma3mz@github&quot;</span><span class="punctuation">,</span></span><br><span class="line">     <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line"> <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line"> <span class="attr">&quot;nodeAttrs&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">     <span class="punctuation">&#123;</span></span><br><span class="line">         <span class="attr">&quot;target&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="string">&quot;group:can-funnel&quot;</span><span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">         <span class="attr">&quot;attr&quot;</span><span class="punctuation">:</span>   <span class="punctuation">[</span><span class="string">&quot;funnel&quot;</span><span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">     <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line"> <span class="punctuation">]</span><span class="punctuation">,</span></span><br></pre></td></tr></table></figure>
+<h3 id="step-2-1">Step 2</h3>
+<p>此时，需要在A机器上开启服务。申请<a
+href="https://tailscale.com/kb/1153/enabling-https/">HTTPS证书</a>，下面的命令会提示后面需要加入什么域名。按照提示复制再执行即可。可能会有一些报错，需要根据不同情况进行更改。该功能可能会影响提供公网服务，但不影响组网服务。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale cert</span><br></pre></td></tr></table></figure>
+<p>在开放服务前有两个命令需要进行区分，<code>tailscale serve</code>和<code>tailscale funnle</code>前者是开放服务到组网环境，后者是开放到整个互联网环境。</p>
+<p>常用有两个技巧</p>
+<ul>
+<li>把已经开放服务的端口转发到其他端口，或者主域名下。比如这里可以把5000端口转发到主域名下的/api路由下，即可以通过
+<code>http://A/api</code>访问A机器的5000端口服务。<code>--bg</code>表示后台运行，默认是在前台启动，便于<code>Ctrl+C</code>关闭。</li>
+</ul>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve --<span class="built_in">bg</span> --set-path /api http://127.0.0.1:5000/</span><br></pre></td></tr></table></figure>
+<p>把 4747 端口转发到 8443 端口，即可以通过
+<code>https://A:8443</code>访问A机器的 4747 端口服务。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve https:8443 / http://localhost:4747</span><br></pre></td></tr></table></figure>
+<ul>
+<li>部署静态服务</li>
+</ul>
+<p>假设本地有一个<code>/data/index.html</code>文件，想要开放访问。则运行下面的命令，可以通过<code>http://A/index</code>访问A机器的<code>/data/index.html</code>文件。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve --<span class="built_in">bg</span> --set-path /index /data/index.html</span><br></pre></td></tr></table></figure>
+<p>如果<code>index.html</code>文件用了本机其他静态资源，同样需要进行挂载，比如挂载<code>/data/static</code>文件夹</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve --<span class="built_in">bg</span> --set-path /static/ /data/static</span><br></pre></td></tr></table></figure>
+<p>如果要关闭服务，可以使用下面的命令，仅关闭这一条。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale serve --<span class="built_in">bg</span> --set-path /api off</span><br></pre></td></tr></table></figure>
+<p>如果希望指定端口，则可以用下面命令，默认端口为 80。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 带端口开放</span></span><br><span class="line">tailscale serve --<span class="built_in">bg</span> --http 10001 --set-path /index /data/index.html</span><br><span class="line"><span class="comment"># 带端口关闭</span></span><br><span class="line">tailscale serve --<span class="built_in">bg</span> --http 10001 --set-path /index off</span><br></pre></td></tr></table></figure>
+<h3 id="step-3-1">Step 3</h3>
+<p>对外开放服务，下面的命令会直接把原本仅能在局域网内访问的服务提供到公网。实测下载速度200-300k/s，有些地方不能直接访问，可能要挂梯子。目前
+tailscale 仅支持三个端口可以dui外开放，分别是 443、8443、10000。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tailscale funnel --<span class="built_in">bg</span> https+insecure://localhost:443</span><br><span class="line">tailscale funnel --<span class="built_in">bg</span> https+insecure://localhost:8443</span><br><span class="line">tailscale funnel --<span class="built_in">bg</span> https+insecure://localhost:10000</span><br></pre></td></tr></table></figure>
+<p>使用方法同 <code>serve</code>，换成 <code>funnel</code> 即可。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看状态的命令</span></span><br><span class="line">tailscale funnel status</span><br><span class="line"><span class="comment"># 关闭状态的命令</span></span><br><span class="line">tailscale funnel 443 off</span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
-        <category>Life</category>
+        <category>Software</category>
       </categories>
       <tags>
-        <tag>Raspberry</tag>
-        <tag>Linux</tag>
-        <tag>Android</tag>
-        <tag>Windows</tag>
-        <tag>IOS</tag>
-        <tag>Kindle</tag>
+        <tag>Tailscale</tag>
       </tags>
   </entry>
   <entry>
-    <title>用ADB调试安卓手机</title>
-    <url>/hexo_blog/2018/01/25/IOT/%E7%94%A8ADB%E8%B0%83%E8%AF%95%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA/</url>
-    <content><![CDATA[<p>使用adb工具实现对安卓手机的调试，包括安装应用、查看运行App、录屏、截图等功能。本文详细介绍了adb工具的安装、配置和使用方法，并列出了一些实用的命令，同时还介绍了无线网调试设备的方法。</p>
+    <title>待整理的脚本（持续更新ing）</title>
+    <url>/hexo_blog/2023/11/26/%E5%BE%85%E6%95%B4%E7%90%86%E8%84%9A%E6%9C%AC/</url>
+    <content><![CDATA[<p>记录一些常用/重要，但容易忘记的命令</p>
 <span id="more"></span>
-<p>写在开头，一些推荐阅读的博客文章</p>
-<p><a
-href="http://blog.csdn.net/fasfaf454/article/details/51438743?locationNum=14">通过adb获取手机信息</a></p>
-<p><a href="https://www.jianshu.com/p/5980c8c282ef">Android
-adb你真的会用吗?</a></p>
-<p><a
-href="http://blog.csdn.net/qq_15364915/article/details/52369266">ADB调试命令大全</a></p>
-<h3 id="介绍">介绍</h3>
-<p>adb是安卓开发调试工具，接触到这个还是因为微信跳一跳的外挂原因。看文档说明之后得知需要adb工具，进行简单尝试后，发现这玩意还真的是挺好用的。填补了自己用代码控制安卓手机的空缺（之前也玩过按键精灵，但是需要Root权限）。</p>
-<p><a
-href="https://developer.android.com/studio/command-line/adb.html?hl=zh-cn"><strong>adb官方说明</strong></a>：</p>
-<p>Android 调试桥 (adb)
-是一个通用命令行工具，其允许您与模拟器实例或连接的 Android
-设备进行通信。它可为各种设备操作提供便利，如安装和调试应用，并提供对
-Unix
-shell（可用来在模拟器或连接的设备上运行各种命令）的访问。该工具作为一个客户端-服务器程序，包括三个组件：</p>
-<h3 id="安装">安装</h3>
-<p>可参考文章: <a
-href="https://www.xda-developers.com/install-adb-windows-macos-linux/">How
-to Install ADB on Windows, macOS, and Linux</a></p>
-<h4 id="windows">Windows</h4>
-<p>这里我使用的是win10。如果以下操作不能成功配置，可以百度其他的安装教程</p>
-<ol type="1">
-<li><p>下载。<a
-href="https://dl.google.com/android/repository/platform-tools-latest-windows.zip">下载链接</a></p></li>
-<li><p>移动目录</p>
-<ol type="1">
-<li>解压压缩包</li>
-<li>移动文件夹你想让它放置的位置</li>
-</ol></li>
-<li><p>环境配置</p>
-<ol type="1">
-<li>添加环境变量，这样可以在cmd或者powershell中直接使用adb</li>
-<li>右键"计算机"-&gt;"属性"-&gt;左边的"高级系统设置"-&gt;"环境变量"&gt;-在上面用户变量中的"Path"进行编辑-&gt;"添加"刚刚放置的文件夹目录，确认保存即可
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135305187.png" /></li>
-<li>这样再打开cmd或者powershell，就可以直接使用adb命令了</li>
-</ol></li>
-</ol>
-<h4 id="linux">Linux</h4>
-<p>Linux系统安装adb工具还是很容易的，我使用的是Ubuntu17.10</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> apt updagte</span><br><span class="line">&gt; <span class="built_in">sudo</span> apt upgrade</span><br><span class="line">&gt; <span class="built_in">sudo</span> apt install android-tools-adb</span><br></pre></td></tr></table></figure>
-<h3 id="配置">配置</h3>
-<p>手机需要打开USB调试模式，在实验的时候，市面上流行的大部分安卓手机USB调式模式开关默认关闭。当然如果手机默认开启了USB调试开关，自然可以省去下面一步。这里简单介绍一下如何手动打开USB调试模式开关，手机“设置”—&gt;"关于手机"—&gt;点击"版本号"7次。每个手机开启的方式大同小异，如果有不同，可以百度“xx手机如何开启USB调试”。</p>
-<p>在USB调试开关开启之后，进入USB调试，打开USB调试开关，如下图。</p>
-<p><img src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135319543.png" width="300px" /></p>
-<p>之后使用USB数据线连接到电脑。</p>
-<p>连接成功后，手机需要信任电脑。电脑打开cmd/powershell（Windows）或者终端（Linux），输入
-<code>adb devices</code>。查看输出。</p>
-<p>如果有输出设备名且无警告，则表示连接成功，否则需要进行下面一步的配置。</p>
-<p><strong>Windows</strong></p>
-<p>首先需要连接手机，确保手机能连接到电脑且打开了USB调试。</p>
-<p>第二步，右键"计算机"-&gt;"属性"-&gt;左边的"设备管理器"-&gt;"便携设备"-&gt;"详细信息"-&gt;"属性"下拉选择"硬件Id"，查看"值"中的内容。</p>
-<p>第三步，进入C盘用户目录下的隐藏文件夹".android"，编辑或者新建文件"adb_usb.ini"，用记事本打开，在里面写上内容。我这里需要写的是"0x2A45"，0x是前缀不需要更改，后面的字符串对应的更改"硬件Id"里面"值"的VID后面的四个字符。保存退出即可</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135348748.png" /></p>
-<p>第四步，断开USB连接，重新连接，再输入
-<code>adb devices</code>。正常情况下应该会出现设备的成功连接的信息</p>
-<p><strong>Linux</strong></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 先连接手机，输入下面命令，找到关于连接手机的那一行</span></span><br><span class="line">&gt; lsusb</span><br><span class="line"><span class="comment"># 输出形如下面，最后的&quot;Google Inc&quot;表示的是设备名，这里需要找到匹配自己手机的设备，并记录好ID</span></span><br><span class="line">...</span><br><span class="line">Bus 001 Device 017: ID 18d1:d002 Google Inc.</span><br><span class="line">...</span><br><span class="line"><span class="comment"># 在目录/etc/udev/rules.d/下，添加文件70-android.rules(名字貌似无所谓？？？)</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> vim /etc/udev/rules.d/70-android.rules</span><br><span class="line"><span class="comment"># 写入下面内容，idVendor表示ID冒号前的数字，idProduct表示的是ID冒号后的数字，MODE固定为&quot;0666&quot;，其他不变</span></span><br><span class="line">SUBSYSTEM==<span class="string">&quot;usb&quot;</span>, ATTRS&#123;idVendor&#125;==<span class="string">&quot;18d1&quot;</span>, ATTRS&#123;idProduct&#125;==<span class="string">&quot;d002&quot;</span>,MODE=<span class="string">&quot;0666&quot;</span></span><br><span class="line"><span class="comment"># 保存退出，修改权限</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> <span class="built_in">chmod</span> 644   /etc/udev/rules.d/70-android.rules</span><br><span class="line">&gt; <span class="built_in">sudo</span> <span class="built_in">chown</span> root. /etc/udev/rules.d/70-android.rules</span><br><span class="line"><span class="comment"># 重启服务</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> service udev restart</span><br><span class="line">&gt; <span class="built_in">sudo</span> killall adb</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输入adb devices查看输出，是否有误</span></span><br><span class="line">&gt; adb devices</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h3 id="使用">使用</h3>
-<p>详细可参考这篇博客: <a
-href="http://blog.csdn.net/qq_15364915/article/details/52369266">ADB调试命令大全</a></p>
-<p>在这里贴出一些我认为比较有意思且实用的功能</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看所有连接设备</span></span><br><span class="line">&gt; adb devices</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行截图保存在sd卡的根目录下，名字为screen.png</span></span><br><span class="line">&gt; adb shell screencap -p /sdcard/screen.png</span><br><span class="line"><span class="comment"># 将截图发送到本地（当前目录下），也可以发送其他文件</span></span><br><span class="line">&gt; adb pull /sdcard/screen.png</span><br><span class="line"><span class="comment"># 删除本地文件</span></span><br><span class="line">&gt; adb shell <span class="built_in">rm</span> /sdcard/screen.png</span><br><span class="line"><span class="comment"># 发送电脑里的文件到设备</span></span><br><span class="line">&gt; adb shell push screen.png /sdcard/</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入手机的交互环境，操作类似linux终端，exit或者Ctrl+C退出</span></span><br><span class="line">&gt; adb shell</span><br><span class="line"></span><br><span class="line"><span class="comment"># 点击手机屏幕(1000,1000)的位置</span></span><br><span class="line">&gt; adb shell input tap 1000 1000</span><br><span class="line"><span class="comment"># 输入字符串&quot;helloworld&quot;，此处不能直接输入中文，且字符串不能有空格</span></span><br><span class="line">&gt; adb shell input text helloworld</span><br><span class="line"><span class="comment"># 滑动屏幕，从（100, 100)到（1000,1000)，经历10s(也可以当作长按屏幕来使用)</span></span><br><span class="line">&gt; adb shell input swipe 100 100 1000 1000 10</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看当前运行的App, 这里Windows没有grep所以会运行失败，可以进入先进入交互环境再输入下面去掉&quot;adb shell&quot;命令</span></span><br><span class="line">&gt; adb shell dumpsys window | grep mCurrentFocus</span><br><span class="line"><span class="comment"># 或者</span></span><br><span class="line">&gt; adb shell dumpsys activity activities | grep mFocusedActivity</span><br><span class="line"></span><br><span class="line"><span class="comment"># 按下电源键</span></span><br><span class="line">&gt; adb shell input keyevent 26</span><br><span class="line"><span class="comment"># 按下返回键</span></span><br><span class="line">&gt; adb shell input keyevent 4</span><br><span class="line"><span class="comment"># 按下HOME健</span></span><br><span class="line">&gt; adb shell input keyevent 3</span><br><span class="line"><span class="comment"># 点亮屏幕</span></span><br><span class="line">&gt; adb shell input keyevent 224</span><br><span class="line"><span class="comment"># 熄灭屏幕</span></span><br><span class="line">&gt; adb shell input keyevent 223</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机安装了哪些App，输出按行输出App的包名</span></span><br><span class="line">&gt; adb shell pm list packages</span><br><span class="line"><span class="comment"># 加&quot;-s&quot;表示只输出系统应用</span></span><br><span class="line"><span class="comment"># 加&quot;-3&quot;表示只输出第三方应用</span></span><br><span class="line"><span class="comment"># 加字符串表示过滤应用名称，当然也可以使用grep</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装apk</span></span><br><span class="line">&gt; adb install &lt;packagename&gt;</span><br><span class="line"><span class="comment"># 卸载apk</span></span><br><span class="line">&gt; adb uninstall &lt;packagename&gt;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 从桌面启动app</span></span><br><span class="line">&gt; adb shell monkey -p &lt;packagename&gt; -c android.intent.category.LAUNCHER 1</span><br><span class="line"><span class="comment"># 关闭app</span></span><br><span class="line">&gt; adb shell am force-stop &lt;packagename&gt;</span><br></pre></td></tr></table></figure>
-<h3 id="使用无线网调试设备">使用无线网调试设备</h3>
-<p>确保手机和电脑在同一局域网内</p>
-<p>如何查看手机的ip，由于每个设备的方法不大相同，在此不进行说明，请读者自行百度。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用USB连接设备，启动手机5555端口</span></span><br><span class="line">&gt; adb tcpip 5555</span><br><span class="line"><span class="comment"># adb连接命令，需要知道手机的ip</span></span><br><span class="line">&gt; adb connect 192.168.1.111</span><br><span class="line"><span class="comment"># 输入如下则表示连接成功</span></span><br><span class="line">connected to 192.168.1.111:5555</span><br><span class="line"><span class="comment"># 这里要注意的是，如果接下来的步骤未断掉数据线，则电脑是连接了两台设备的，进行操作会产生错误，所以可以拔掉数据线或者断开网络或者指定设备运行</span></span><br><span class="line"><span class="comment"># 断开无线网连接</span></span><br><span class="line">&gt; adb disconnect 192.168.1.111</span><br></pre></td></tr></table></figure>
+<h3 id="git-统计不同版本的文件夹代码行数">Git
+统计不同版本的文件夹代码行数</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git checkout &lt;commit_id&gt;</span><br><span class="line">find -name <span class="string">&quot;*.py&quot;</span> | xargs <span class="built_in">cat</span> | <span class="built_in">wc</span> -l</span><br></pre></td></tr></table></figure>
+<h3 id="git-撤销-commit">Git 撤销 commit</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git revert &lt;commit_id&gt;</span><br></pre></td></tr></table></figure>
+<h3 id="git-撤销单个文件的修改">Git 撤销单个文件的修改</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git checkout HEAD -- &lt;文件路径&gt;</span><br></pre></td></tr></table></figure>
+<h3 id="git-暂存">Git 暂存</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">git stash</span><br><span class="line">git stash pop</span><br></pre></td></tr></table></figure>
+<h3 id="grep-搜索文本并显示文件名和行号">Grep
+搜索文本并显示文件名和行号</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">grep -Hn <span class="string">&quot;Hello World!&quot;</span> *</span><br></pre></td></tr></table></figure>
+<h3 id="grep-搜索不包含某个字符串的文本并显示文件名和行号">Grep
+搜索不包含某个字符串的文本并显示文件名和行号</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">grep -Hn <span class="string">&quot;Hello World!&quot;</span> * | grep -v <span class="string">&quot;XXX&quot;</span></span><br></pre></td></tr></table></figure>
+<h3 id="python-注册全局变量以在所有代码中使用">Python
+注册全局变量以在所有代码中使用</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="built_in">globals</span>()[<span class="string">&quot;seed&quot;</span>] = <span class="number">42</span></span><br></pre></td></tr></table></figure>
+<h3 id="python-动态修改源码-monkey-patching">Python 动态修改源码 (Monkey
+Patching)</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在源码中</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">A</span>:</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self</span>):</span><br><span class="line">        ...</span><br><span class="line"></span><br><span class="line"><span class="comment"># 在项目中</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">new_forward</span>(<span class="params">self</span>):</span><br><span class="line">    ...</span><br><span class="line">A.forward = new_forward</span><br></pre></td></tr></table></figure>
+<h3 id="wget-下载文件保存为时间戳">wget 下载文件，保存为时间戳</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget -O `<span class="built_in">date</span> +%s`.jpg https://www.baidu.com/img/bd_logo1.png</span><br></pre></td></tr></table></figure>
+<h3 id="nginx-端口转发">Nginx 端口转发</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">server &#123;</span><br><span class="line">    listen 4747;</span><br><span class="line">    server_name localhost;</span><br><span class="line">    proxy_pass http://x.x.x.x:4747;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h3 id="linux-统计当前路径下的文件个数">Linux
+统计当前路径下的文件个数</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">ls</span> -l | grep <span class="string">&quot;^-&quot;</span> | <span class="built_in">wc</span> -l</span><br></pre></td></tr></table></figure>
+<h3 id="kill-python-相关进程">kill Python 相关进程</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">ps -ef | grep python | grep -v grep | awk <span class="string">&#x27;&#123;print $2&#125;&#x27;</span> | xargs <span class="built_in">kill</span> -9</span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
-        <category>Android</category>
+        <category>Linux</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>ADB</tag>
+        <tag>Note</tag>
       </tags>
   </entry>
   <entry>
-    <title>Birthday in Pi：查找日期在Pi中的位置</title>
-    <url>/hexo_blog/2021/03/18/Python/Birthday_in_Pi/</url>
-    <content><![CDATA[<p>想知道自己的生日在圆周率π的第几位出现吗？本文介绍了从思路到实现所遇到的问题，详细分析了如何使用迭代器优化性能，并改用Python语言实现了类似“
-锁”内存等操作。最终，将函数打包成了exe程序，在公众号内提供查询服务，还提供了一个不断更新的生日与位置的查表。</p>
+    <title>本科科研的那些事</title>
+    <url>/hexo_blog/2018/09/30/%E6%9C%AC%E7%A7%91%E7%A7%91%E7%A0%94%E7%9A%84%E9%82%A3%E4%BA%9B%E4%BA%8B/</url>
+    <content><![CDATA[<p>本科科研经历不仅对建立人脉、提升简历很重要，对发现兴趣、培养批判性思维能力也很重要。加入一个有可靠的导师和同行的合适的研究实验室，不仅可以帮助提高一个人的学术技能，还可以提供接触科学方法的机会，并帮助明确职业目标。本文仅叙说个人经历，从选择研究课题到制定研究方法，以及学习文献研究、编程和硬件等实用技能的重要性。作者还强调了在研究工作中追求科学兴趣和独立的重要性。</p>
 <span id="more"></span>
-<p>完整的项目代码均开源: <a
-href="https://github.com/wnma3mz/birthday_in_pi">https://github.com/wnma3mz/birthday_in_pi</a></p>
-<h2 id="想法">想法</h2>
-<p>于我而言，很多项目的出发点都是源于浏览东西时所想，觉得有意思就去实现。比如这次是在即刻上看到北大数院公众号，可以回复生日（如20000101）就可以知道这串数字在Pi的第多少位出现。更详细的例子：已知3.1415926...，那么1415就是出现在第1位（从1开始计数，小数点后开始计数）。但该公众号仅在Pi的2亿位数中进行查找，所以有即友回复，搜不到结果。故想到可以试试更长的数据集是否能够搜到。</p>
-<p><strong>背景知识</strong></p>
-<p>理论上，Pi里面可以查找所有人的电话号码、生日、银行卡密码等等。</p>
-<h2 id="行动">行动</h2>
-<h3 id="数据">数据</h3>
-<p>巧妇难为无米之炊，这里最重要的就是Pi这个无理数数据。</p>
-<ol type="1">
-<li>是否有现成的数据文本-&gt;没搜到，搜到的位数都很小，不足以满足需求</li>
-<li>计算Pi（生成超过2亿数的文本）
-<ol type="1">
-<li>自己手写</li>
-<li>找程序。对于计算Pi的项目，本身就是一个科学问题，故一定有现有程序可以辅助完成。所以找到了<a
-href="http://www.numberworld.org/y-cruncher/">y-cruncher</a>项目。具体是否为当前最快的生成程序，未深究，不过实际测试，效率已经可以满足需求。（用i5-1035G1的CPU，生成25亿数，8个线程计算用了10分钟不到，文本大小2.4G）</li>
-</ol></li>
-</ol>
-<h3 id="编码">编码</h3>
-<p>该问题本身简单来看就是一个字符串查找问题，所以用Python的话，<code>find</code>函数可以直接完成这项任务。但一旦问题变"大"（数据集变大、机器变多......），就是一个"难"题了。</p>
-<p>当然，先不管如何，试试最粗暴的解法看看性能如何。根据测试结果，有两个问题，时间超过5s,
-内存占用2G（文本大小2G）。就算时间可以接受，那么这个空间占用也很难接受。试想，每查询一次都需要占用2G内存，查询完再释放。所以直接想到的优化点有以下几个</p>
-<ol type="1">
-<li>"锁"内存，即每次查询完不释放这个内存，这样IO时间就可以大大减小</li>
-<li>"做表"，查询的生日数字是有限的，无非就是(19700101-20101231有限数据)，就算100年，也只是36500个数据（实际会略高，因为有闰年）。查表的效率就非常快了</li>
-<li>从根本上优化这个程序</li>
-</ol>
-<h3 id="优化">优化</h3>
-<p>首先排除2，因为最终还是想查找任意数字，2不便于后续扩展，而且根据现有的性能，做表也需要很长时间。</p>
-<p>其次对于1，如果考虑对外开放，服务器占用2G内存空间，实在是太"奢侈"了</p>
-<p>所以最后就只能考虑3。</p>
-<h4 id="迭代器">迭代器</h4>
-<p>Python有个关键词是
-<code>yield</code>，它可以实现迭代器这个功能。说人话就是，它不是一次性把数据加载出来，而且每次只加载一部分。下面是一个简单的例子</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">yield_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> lst:</span><br><span class="line">        <span class="keyword">yield</span> i</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">for_return_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> lst:</span><br><span class="line">        <span class="keyword">return</span> i</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">return_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">return</span> lst</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    lst = <span class="built_in">list</span>(<span class="built_in">range</span>(<span class="number">10</span>))</span><br><span class="line">    <span class="built_in">print</span>(yield_func(lst))</span><br><span class="line">    <span class="built_in">print</span>(for_return_func(lst))</span><br><span class="line">    <span class="built_in">print</span>(return_func(lst))</span><br><span class="line">    <span class="keyword">for</span> x <span class="keyword">in</span> yield_func(lst):</span><br><span class="line">        <span class="built_in">print</span>(x, end=<span class="string">&quot; &quot;</span>)</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 输出如下</span></span><br><span class="line">&lt;generator object yield_func at 0x000001644AF624F8&gt;</span><br><span class="line">0</span><br><span class="line">[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]</span><br><span class="line">0 1 2 3 4 5 6 7 8 9</span><br></pre></td></tr></table></figure>
-<p>所以肯定是要利用这个方法来帮助优化程序。在Python操作文件的时候
-<code>readline()</code>函数就是一个生成器，一行一行的返回，查找速度比直接
-<code>read()</code>或者 <code>readlines()</code>更快。</p>
-<p>但这里的问题是，输出的Pi文本是一行。。。所以没法直接使用。当然也可以手动把这个文本按行切割后，再用
-<code>readline()</code>函数。</p>
-<p>这里我找到的解决方案[1]可以更方便的解决这个问题，详见博客or项目源码。</p>
-<h4 id="bug">bug</h4>
-<p>把程序写好之后，可以发现一个很明显的bug，就是查询的字符串如果在两个
-<code>chunk</code>之间，是会直接跳过。比如
-<code>12345678</code>，<code>block_size=4</code>，那么就是
-<code>1234</code>和 <code>5678</code>。如果查找
-<code>45</code>是找不到的。</p>
-<p>所以这里需要做两个额外的处理</p>
-<ol type="1">
-<li>加上上一个 <code>block_size</code>的字符串
-<code>old_chunk = copy.deepcopy(chunk[-len(target_str) :])</code>。这里为了省空间和时间，只加上了需要查询字符串的长度，就可以满足需求</li>
-<li>返回查找位数时需要额外
-<code>-len(old_chunk)</code>。（注：这里一开始忘了操作，故前几位查询的既有会发现结果与北大数院查询的结果差10位，在这里致歉）</li>
-</ol>
-<p>最后经过测试后，性能基本达到预期（见项目README，有说明具体结果）。</p>
-<h2 id="部署">部署</h2>
-<p>至此，核心程序已经完成，接下来需要考虑一种方式来给其他人方便使用了。最直接的其实就是写一个Web，前端去调用后端返回结果。但同样的，这里还是考虑到网站流量等原因，跟北大数院一样放到公众号上也很方便，开发上也只需要完成后端即可，无需考虑前端UI啥的。</p>
-<p>查询公众号文档后，一通操作，就把功能完成了（文案直接"抄"了北大数院的文案）。代码见<a
-href="https://github.com/wnma3mz/birthday_in_pi/blob/master/flask_server.py">flask_server.py</a></p>
-<p>这里需要在微信公众号后台配置开发者模式，然后把token复制进代码。</p>
-<h3 id="tip">Tip</h3>
-<p>公众号开发要求是80端口，但这里通过子域名的方式+nginx反代就可以直接用域名访问，而无需80端口。nginx配置见下</p>
-<figure class="highlight nginx"><table><tr><td class="code"><pre><span class="line"><span class="section">server</span> &#123;</span><br><span class="line">    <span class="attribute">listen</span>       <span class="number">80</span>;</span><br><span class="line">    <span class="attribute">server_name</span>  xxx.yyy.zzz;</span><br><span class="line"></span><br><span class="line">    <span class="section">location</span> / &#123;</span><br><span class="line">        <span class="attribute">uwsgi_pass</span> <span class="number">0.0.0.0</span>:port;</span><br><span class="line">        <span class="attribute">proxy_pass</span> http://0.0.0.0:port/wx_api;</span><br><span class="line">    &#125;</span><br><span class="line">  </span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
-<p>这里的 <code>xxx.yyy.zzz</code>就是域名，port就是
-<code>flask</code>开启的端口</p>
-<h2 id="其他">其他</h2>
-<h3 id="查表">查表</h3>
-<p>当然，查表的方式还是最快的，所以还是建了一个表，并且做了一个简单的分析。分析内容如下</p>
-<p>最近只需246位能够找到,
-最远需要近9亿位。列出Top10（左侧为年月日，右侧为第x位）</p>
-<p>日期范围为19700101-20091231</p>
-<table>
-<thead>
-<tr>
-<th>日期</th>
-<th>位数</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>20190914</td>
-<td>246</td>
-</tr>
-<tr>
-<td>20190516</td>
-<td>5364</td>
-</tr>
-<tr>
-<td>19910403</td>
-<td>8365</td>
-</tr>
-<tr>
-<td>20141020</td>
-<td>9813</td>
-</tr>
-<tr>
-<td>19950728</td>
-<td>1,3874</td>
-</tr>
-<tr>
-<td>20121031</td>
-<td>1,4538</td>
-</tr>
-<tr>
-<td>19990814</td>
-<td>2,5593</td>
-</tr>
-<tr>
-<td>19810705</td>
-<td>2,7202</td>
-</tr>
-<tr>
-<td>19921122</td>
-<td>3,0884</td>
-</tr>
-<tr>
-<td>19930509</td>
-<td>3,6615</td>
-</tr>
-<tr>
-<td>..........</td>
-<td>..........</td>
-</tr>
-<tr>
-<td>19941227</td>
-<td>7,1993,2960</td>
-</tr>
-<tr>
-<td>20080923</td>
-<td>7,2930,2729</td>
-</tr>
-<tr>
-<td>19911201</td>
-<td>7,2985,0775</td>
-</tr>
-<tr>
-<td>20071206</td>
-<td>7,3782,1997</td>
-</tr>
-<tr>
-<td>20090220</td>
-<td>7,5402,4215</td>
-</tr>
-<tr>
-<td>19730829</td>
-<td>7,7357,0338</td>
-</tr>
-<tr>
-<td>20070526</td>
-<td>7,8655,8038</td>
-</tr>
-<tr>
-<td>19951122</td>
-<td>8,2582,0139</td>
-</tr>
-<tr>
-<td>20140407</td>
-<td>8,7320,8091</td>
-</tr>
-<tr>
-<td>19730521</td>
-<td>8,8950,7619</td>
-</tr>
-</tbody>
-</table>
-<p>为什么4位分隔，而不是3位？</p>
-<p>因为我觉得这样看的更舒服，读万、亿更方便。。。</p>
-<h3 id="程序打包">程序打包</h3>
-<p>为方便没有Python程序的用户，同时也不想使用公众号。在Windows上打包了一份exe程序。命令也很简单
-<code>pyinstaller -F .\read_large_pi.py</code>。</p>
-<p>最后的最后，该程序尚存在优化空间，当查询数字较长（超过8位）时，时间花费也更久（超过1s）。</p>
-<h2 id="参考文章">参考文章</h2>
-<p>[1] <a
-href="https://lovemma.github.io/posts/python-%E5%A6%82%E4%BD%95%E6%B5%81%E5%BC%8F%E8%AF%BB%E5%8F%96%E5%A4%A7%E6%96%87%E4%BB%B6/">python-如何流式读取大文件</a></p>
+<h2 id="本科科研的那些事">本科科研的那些事</h2>
+<p>本科阶段，以就业为目的的同学加入一个实验室其实是无所谓的，除非是需要倚重相关老师。其他无论是考/保研，还是出国留学的同学，选择并加入一个合适的实验室还是十分重要的。</p>
+<p>一个好的实验室往往会有靠谱导师和若干师兄师姐。从建立人脉网络上来看，导师可能会在你部分专业课程上帮助你打出高分，在考/保研阶段能够帮你推荐导师。师兄师姐也有庞大的校友资源，所以起码在信息资源上就比其他人更领先一筹。再退一步来说，进入实验室也可以为本科阶段的简历添上一笔。亦或是，为迷茫的同学更进一步了解读研的工作生活，了解清楚科研到底是什么，如何进行科研，所属专业内的学术圈子是如何如何的。</p>
+<p>就我个人而言，大二下时加入了一个新成立且庞大的实验室。大概工作就是从零造一台医疗仪器，所以学生来源于各个专业，机械、电子、材料、计算机等。各自专业的分工也不同。我被分配的课题是"大规模系统响应矩阵的快速分解"，简单来说就是对矩阵进行分解，只不过现在这个矩阵规模很大，在分解的时候我们还要追求速度。</p>
+<p>所以当一个问题抛出来时，在没有外界帮助的前提下，如何解决这个问题。我自己的思考与解决步骤如下：直接在知网中搜索这一长串名词，进行简单的文献阅读后，提取得到”矩阵分解“这一关键词。再分别用中英文对这一关键词进行搜索，再进行二次学习。最终要达到能够比较清楚的解释这是什么，目前的研究进展（简要了解），目前的研究方法（简要了解）等。这基本就属于论文中的
+<code>Introduce</code>部分。</p>
+<p>下一步，如何解决这个问题。就拿矩阵分解这一问题来说，搜索矩阵分解这一方法，很容易发现有很多方法，其中QR、LU、SVD是用的比较多。这时候就需要根据问题的具体需求（速度、准确度...）来选择合适的方法。选定方法后，就需要考虑是改进现有的方法（数学算法上）还是从其他角度入手。显然，在改进数学算法上难度还是摆在那里的。所以这里考虑一些其他方法，其他问题也可以以类似思路来解决。首先，这个问题我们是用计算机来解决的，那么显然可以想到两点，编程上进行加速、硬件上进行加速。硬件上加速简单来说就是用最好的硬件平台（计算资源），当然这样基本不具任何创新性。还可以使用诸如分布式计算这种方法来加速，这样问题就可以转换为如何将矩阵分解应用到分布式系统中，如何加速，这样问题就成功转换了。另一点，在程序上进行加速，这里就需要引入几个简单的名词：并发、并行、异步、流式计算。当然还有一些其他技巧就不展开了。这里我选用的是并行计算，那么新的问题就是如何进行矩阵分解的并行计算，是使用何种框架何种语言等等。这大概是我整个的一个思考思路，当然很多思路也可以从其他文献的阅读中进行获取。还有一个点就是，可以多种方法结合，比如这里可以做分布并行计算，当然这一点其实创新性其实也比较差，毕竟都是现有方法，除非有一些不同与这两个方法的点。</p>
+<p>所以，不仅可以学到这种思考方式，还在其中可以学习到其他技巧，比如如何搜索并下载文献，如何阅读文献等。再说回自己，由于实验室历史沉淀不足，所以很多东西都需要从零开始并自己动手。所以需要从零选购计算机配件，组装，搭建环境，独立编程等。以上可能老师也没时间来指导，基本上也都是给一个大方向没有比较具体的解决方案。所以基本都是自己或跟小伙伴一起完成。从中收获到的自然是学习到了独当一面的能力，自己动手能力又提高了；但缺点也很明显，学习时间成本是巨大的，并且老师可能会让你跑腿做其他事情。所以在这里建议，这种实验室还是慎入，当然如果为了锻炼自己一下，也可以试试，毕竟最后成就感也是十足的。我自己最后大概就是将QR、LU、SVD这三种算法进行了一个复现，SVD是用CUDA完成的。<a
+href="https://github.com/wnma3mz/Matrix-factorization">链接</a>。基于种种原因，后续并没有进行下去，即没有成果的产出，不过学到了东西才是最重要的（23333）。</p>
+<p>再谈本学院科研训练，基本也是类似于加入一个实验室这种。这里不同于上面的点在于，老师在这里基本会给出一个比较成熟的方案给予复现，所谓材料约等于搬砖（炒菜同理）。相反在这里学习到的东西其实并不算多，只是实验（非计算机）的操作步骤，仪器的使用，说白了技术性真的不算高。但主要还是方案的提出与方法的改进这种。最后的成果也是一篇SCI，其余倒真的没什么好说。</p>
+<p>就像文章中一开始说的那样，提前进入实验室能够让我认清自己是否能够做科研工作，是否喜欢。就拿科研训练这个来说，通过这个基本也认清了学科的特性，确实也没啥意思，所以算是自主劝退了。关于材料学科的特性以及各种劝退文，我在这不是很想介绍，具体可见知乎。个人还是秉持着理性劝退的态度，认清自己的追求再做决定。再谈那个实验室，就工作内容而言还是可以接受的，但是由于缺乏一定的动力（比如各种都需要自己独立完成，方向上不是自己将来想做的，老师指导时间少），故也选择了退出。不过总的来说，这两次经历确实收获颇多，着实为之后的很多工作打好了一个良好的基础，自己的思考角度也更加全面与丰富了（因人而异）。同时也认清一点就是，很多看似对自己无用的东西，其实后续都有很大的帮助。比如写博客这件小事，无形中对我论文写作提供了一个基础（能够有逻辑、清晰的阐述某件事）；再比如大一时被强制学习的
+<code>LaTex</code>，也十分有效（当然本来用word也行）；再比如SVD矩阵分解，可能当时没有想到一年后的自己还会继续做这个相关内容（本文写作于2019年9月）等。在大部分事情上，还是不要太功利，以兴趣为导向，说不定将来某一天会给予你巨大的帮助。</p>
 ]]></content>
       <categories>
-        <category>Note</category>
+        <category>Life</category>
       </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>微信开发</tag>
-        <tag>Steaming</tag>
-      </tags>
   </entry>
   <entry>
-    <title>Python3的Tools-scripts整理</title>
-    <url>/hexo_blog/2020/06/14/Python/Python3%E7%9A%84Tools-scripts%E6%95%B4%E7%90%86/</url>
-    <content><![CDATA[<p>Python3的Tools-scripts整理</p>
+    <title>杂记</title>
+    <url>/hexo_blog/2018/05/30/%E6%9D%82%E8%AE%B0/</url>
+    <content><![CDATA[<p>SimHash算法是一种用于计算文档相似度的算法，可以通过计算文本的SimHash值来比较文档间的汉明距离，从而比较文档间的相似度。本文介绍了SimHash算法的基本原理和实现步骤，并介绍了中文分词方法和不同类型的分词算法。同时，还对损失函数、汉明距离、hash函数、频率与频度、互现信息等关键概念进行了详细解释，有助于读者深入了解SimHash算法背后的技术原理。</p>
 <span id="more"></span>
-<p>文件路径为<code>Anaconda3\Tools\scripts</code></p>
-<h3 id="byext.py">byext.py</h3>
-<p>分析文件/文件夹</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># py文件</span></span><br><span class="line">ext  <span class="built_in">bytes</span>  files  lines  words</span><br><span class="line">.py   <span class="number">2655</span>      <span class="number">1</span>     <span class="number">91</span>    <span class="number">265</span></span><br><span class="line">TOTAL   <span class="number">2655</span>      <span class="number">1</span>     <span class="number">91</span>    <span class="number">265</span></span><br><span class="line">ext  <span class="built_in">bytes</span>  files  lines  words</span><br><span class="line"></span><br><span class="line"><span class="comment"># 二进制文件</span></span><br><span class="line">   ext binary  <span class="built_in">bytes</span>  files</span><br><span class="line">&lt;none&gt;      <span class="number">1</span>  <span class="number">10382</span>      <span class="number">1</span></span><br><span class="line"> TOTAL      <span class="number">1</span>  <span class="number">10382</span>      <span class="number">1</span></span><br><span class="line">   ext binary  <span class="built_in">bytes</span>  files</span><br><span class="line"></span><br><span class="line"><span class="comment"># 文件夹</span></span><br><span class="line">  ext binary  <span class="built_in">bytes</span>   dirs  files  lines  words</span><br><span class="line"> .png      <span class="number">2</span>   <span class="number">2823</span>             <span class="number">2</span></span><br><span class="line">  .py         <span class="number">10471</span>             <span class="number">5</span>    <span class="number">224</span>    <span class="number">647</span></span><br><span class="line"> .txt         <span class="number">17404</span>             <span class="number">3</span>    <span class="number">554</span>    <span class="number">960</span></span><br><span class="line">&lt;<span class="built_in">dir</span>&gt;                    <span class="number">1</span></span><br><span class="line">TOTAL      <span class="number">2</span>  <span class="number">30698</span>      <span class="number">1</span>     <span class="number">10</span>    <span class="number">778</span>   <span class="number">1607</span></span><br><span class="line">  ext binary  <span class="built_in">bytes</span>   dirs  files  lines  words</span><br></pre></td></tr></table></figure>
-<table>
-<thead>
-<tr>
-<th>输出</th>
-<th>主要语法</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>ext</td>
-<td><code>os.path.normcase(os.path.splitext[1])</code></td>
-</tr>
-<tr>
-<td>files</td>
-<td><code>os.path.isfile</code></td>
-</tr>
-<tr>
-<td>bytes</td>
-<td>以二进制(<code>rb</code>)读文件，<code>len</code>的输出</td>
-</tr>
-<tr>
-<td>lines</td>
-<td><code>str(data, 'latin-1').splitlines()</code></td>
-</tr>
-<tr>
-<td>words</td>
-<td><code>len(data.split())</code>的输出</td>
-</tr>
-<tr>
-<td>binary</td>
-<td><code>if *b*'\0' in data</code></td>
-</tr>
-<tr>
-<td>dirs</td>
-<td>os.path.isdir</td>
-</tr>
-<tr>
-<td>lnk</td>
-<td>os.path.islink</td>
-</tr>
-</tbody>
-</table>
-<h3 id="checkpip.py">checkpip.py</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">The latest version of setuptools on PyPI <span class="keyword">is</span> <span class="number">46.2</span><span class="number">.0</span>, but ensurepip has <span class="number">40.8</span><span class="number">.0</span></span><br><span class="line">The latest version of pip on PyPI <span class="keyword">is</span> <span class="number">20.1</span>, but ensurepip has <span class="number">19.0</span><span class="number">.3</span></span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 主要是检查版本</span></span><br><span class="line"><span class="keyword">import</span> ensurepip</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(ensurepip._PROJECTS)</span><br><span class="line"><span class="comment"># [(&#x27;setuptools&#x27;, &#x27;40.8.0&#x27;), (&#x27;pip&#x27;, &#x27;19.0.3&#x27;)]</span></span><br></pre></td></tr></table></figure>
-<h3 id="checkpyc.py">checkpyc.py</h3>
-<p>检查pyc文件是否最新</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> importlib.util</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(importlib.util.MAGIC_NUMBER)</span><br><span class="line"><span class="comment"># b&#x27;B\r\r\n&#x27;</span></span><br></pre></td></tr></table></figure>
-<h3 id="cleanfuture.py">cleanfuture.py</h3>
-<p>修改py文件的future语法</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">errprint</span>(<span class="params">*args</span>):</span><br><span class="line">    strings = <span class="built_in">map</span>(<span class="built_in">str</span>, args)</span><br><span class="line">    msg = <span class="string">&#x27; &#x27;</span>.join(strings)</span><br><span class="line">    <span class="keyword">if</span> msg[-<span class="number">1</span>:] != <span class="string">&#x27;\n&#x27;</span>:</span><br><span class="line">        msg += <span class="string">&#x27;\n&#x27;</span></span><br><span class="line">    sys.stderr.write(msg)</span><br><span class="line">    </span><br><span class="line">errprint(<span class="string">&quot;Usage:&quot;</span>, __doc__)</span><br></pre></td></tr></table></figure>
-<h3 id="crlf.py">crlf.py</h3>
-<p>文件中CRLF用LF替换</p>
-<h3 id="lfcr.py">lfcr.py</h3>
-<p>文件中LF用CRLF替换</p>
-<p>迭代：<code>itertools.count()</code></p>
-<h3 id="reindent.py">reindent.py</h3>
-<p>py文件改为4空格缩进</p>
-<h3 id="untabify.py">untabify.py</h3>
-<p>同上</p>
-<h3 id="texi2html.py">texi2html.py</h3>
-<p>texi to html</p>
-<p><code>import keyword</code></p>
-<p><code>import webbrowser</code></p>
-]]></content>
-      <categories>
-        <category>Python</category>
-      </categories>
-      <tags>
-        <tag>Scripts</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Python杂谈</title>
-    <url>/hexo_blog/2018/03/14/Python/Python%E6%9D%82%E8%B0%88/</url>
-    <content><![CDATA[<p>理解Python中的GIL(Global Interpreter
-Lock)，以及如何使用多线程和多进程，以及弱引用。</p>
-<span id="more"></span>
-<h2 id="gil">GIL</h2>
-<p>建议阅读文章</p>
-<p><a
-href="http://zhuoqiang.me/python-thread-gil-and-ctypes.html">python
-线程，GIL 和 ctypes</a></p>
-<p><a
-href="https://zhuanlan.zhihu.com/p/20953544">谈谈python的GIL、多线程、多进程</a></p>
-<p>GIL, <code>Global Interpreter Lock</code>，全局解释器锁</p>
-<p>产生原因: 防止多线程并发执行机器码的Mutex。</p>
-<p>好处:
-不需要考虑线程安全问题，在单线程的情况下更快(相对而言)，和C库结合时更加方便</p>
-<p>形象的比方:</p>
-<blockquote>
-<p>我们把整个进程空间看做一个车间，把线程看成是多条不相交的流水线，把线程控制流中的字节码看作是流水线上待处理的物品。Python
-解释器是工人，整个车间仅此一名。操作系统是一只上帝之手，会随时把工人从一条流水线调到另一条——这种“随时”是不由分说的，即不管处理完当前物品与否。</p>
-</blockquote>
-<blockquote>
-<p>若没有 GIL。假设工人正在流水线 A 处理 A1 物品，根据 A1
-的需要将房间温度（一个全局对象）调到了 20
-度。这时上帝之手发动了，工人被调到流水线 B 处理 B1 物品，根据 B1
-的需要又将房间温度调到了 50 度。这时上帝之手又发动了，工人又调回 A
-继续处理 A1。但此时 A1 暴露在了 50 度的环境中，安全问题就此产生了。</p>
-</blockquote>
-<blockquote>
-<p>而 GIL 相当于一条锁链，一旦工人开始处理某条流水线上的物品，GIL
-便会将工人和该流水线锁在一起。而被锁住的工人只会处理该流水线上的物品。就算突然被调到另一条流水线，他也不会干活，而是干等至重新调回原来的流水线。这样每个物品在被处理的过程中便总是能保证全局环境不会突变。</p>
-</blockquote>
-<p>在解释器解释python代码时，首先会获得这把锁，需要I/O操作就是释放这把锁。如果是纯计算程序，没有I/O操作，解释器会每隔100次操作释放一次锁（Python3.2之后使用固定时间(5ms)来释放）。</p>
-<p>GIL只会影响严重依赖CPU的程序（计算型）,
-所以运行Python的时候，同一时间只能执行一个线程(单核CPU下的多线程都只是并发)。运行Python程序就只能占用一个核的CPU资源，哪怕使用了
-<code>threading</code>多线程也是一样。</p>
-<p>CPU密集性代码(循环)：不适用多线程。不断触发GIL，线程来回切换，消耗资源</p>
-<p>IO密集型代码(文件处理，网路爬虫)：多线程能够提高效率。在一个线程等待的同时，切换到另一个线程，不浪费CPU资源从而提升效率。</p>
-<p>多核多线程比单核多线程更差，单核下多线程每次释放GIL，唤醒的线程都能获取GIL锁，可以无缝执行。但是多核下，一个CPU释放完GIL之后，所有CPU的线程又开始竞争，可能会被原来的CPU获取，导致其他CPU线程唤醒之后必须等待到下一次释放。这样造成的结果称为线程颠簸(thrashing),
-效率会更低下。</p>
-<p>以上，如果Python需要利用多核CPU，就使用多进程(<code>multiprocessing</code>)，各自拥有独立的GIL，互不干扰，从而达到并行执行。还有一种方案就是使用C进行拓展，手动释放GIL，这里不做介绍。</p>
-<p>参考代码</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-   </span></span><br><span class="line"><span class="keyword">from</span> time <span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">from</span> threading <span class="keyword">import</span> Thread</span><br><span class="line"><span class="keyword">from</span> multiprocessing <span class="keyword">import</span> Process</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">spawn_n_threads</span>(<span class="params">n, target</span>):</span><br><span class="line">    threads = []</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        thread = Thread(target=target)</span><br><span class="line">        thread.start()</span><br><span class="line">        threads.append(thread)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> thread <span class="keyword">in</span> threads:</span><br><span class="line">        thread.join()</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">spawn_n_processes</span>(<span class="params">n, target</span>):</span><br><span class="line">    threads = []</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        thread = Process(target=target)</span><br><span class="line">        thread.start()</span><br><span class="line">        threads.append(thread)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> thread <span class="keyword">in</span> threads:</span><br><span class="line">        thread.join()</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">test</span>(<span class="params">target, number=<span class="number">10</span>, spawner=spawn_n_threads</span>):</span><br><span class="line">    <span class="keyword">for</span> n <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">5</span>):</span><br><span class="line">        start_time = time()</span><br><span class="line">        <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(number):</span><br><span class="line">            spawner(n, target)</span><br><span class="line">        end_time = time()</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Time elapsed with &#123;&#125; branch(es): &#123;:.6f&#125; sec(s)&quot;</span>.<span class="built_in">format</span>(n, end_time - start_time))</span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">fib</span>():</span><br><span class="line">    a = b = <span class="number">1</span></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">100000</span>):</span><br><span class="line">        a, b = b, a + b</span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;threads:&quot;</span>)</span><br><span class="line">    test(fib)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;processes:&quot;</span>)</span><br><span class="line">    test(fib, spawner=spawn_n_processes)</span><br></pre></td></tr></table></figure>
-<p>P.S.</p>
+<h2 id="simhash算法原理">SimHash算法原理</h2>
+<p><code>SimHash</code>是用来计算文档之间的相似度。通过该算法计算每篇文档的
+<code>SimHash</code>值，通过计算得到的二进制来比较文档间的<a
+href="#hanming">汉明距离</a>，进而来比较文档间的相似度。</p>
+<p>步骤：</p>
+<ol type="1">
+<li><p>分词</p>
+<p>对一段文本进行切割，得到有效的特征向量（词）。为每个特征向量设置对应的权值（1-5），权值越高则代表这个特征向量在文本中越重要。</p></li>
+<li><p>hash</p>
+<p>利用 <code>hash</code>函数计算各个特征向量的 <code>hash</code>值。<a
+href="#hash">hash函数</a></p></li>
+<li><p>加权</p>
+<p>在 <code>hash</code>之后，对每个特征向量进行加权，即<strong>W = Hash
+* Weight</strong></p></li>
+<li><p>合并</p>
+<p>在对每个特征向量加权后，再累加，变成一个序列串。</p></li>
+<li><p>降维</p>
+<p>由于最后合并的数字可能过大（过小），设定一个阈值，大于某阈值的设置为1，小于某阈值的设置为0。有很多降维处理的操作，这里只是简单举个例子</p></li>
+</ol>
+<p>举例：</p>
+<p>文本：人生苦短，我用Python</p>
+<ol type="1">
+<li><p>分词。</p></li>
+<li><p>hash。</p></li>
+<li><p>加权。遇到0就变为负数</p></li>
+<li><p>合并。对应位加和。</p></li>
+<li><p>降维。大于0的设置为1，否则为0。</p>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">分词	  hash				加权 			合并			 降维</span><br><span class="line"></span><br><span class="line">生(5)  人生(&quot;10101&quot;)	  	5 -5 5 -5 5	  5+5+4+3-3-3-2=9 	1</span><br><span class="line">ython(5)  Python(&quot;11101&quot;)	5 5 5 -5 5	   -5+5-4-3-3-3-2=-15	0</span><br><span class="line">(4) 	 苦(&quot;10110&quot;)		  4 -4 4 4 -4	 5+5+4+3+3-3-2=15	  1</span><br><span class="line">(3)	 短(&quot;00101&quot;)		 3 -3 3 3 -3	 -5-5+4+3-3-3+2=-7	 0</span><br><span class="line">(3)	 我(&quot;00001&quot;)		 -3 -3 3 -3 3	 5+5-4-3+3+3-2=7	1</span><br><span class="line">(2)  	 用(&quot;00010&quot;)		 -2 -2 -2 2 -2</span><br></pre></td></tr></table></figure></li>
+</ol>
+<h2 id="中文分词方法">中文分词方法</h2>
+<h3 id="基于规则的分词方法">基于规则的分词方法</h3>
+<p>机械分词、基于字典。按照一定的策略在一个词典中的词条进行匹配，若找到某词条则匹配成功。</p>
+<p>三要素：</p>
 <ul>
-<li>并行：同时处理两个及以上的事件在同一时刻发生</li>
-<li>并发：同时处理两个及以上的事件在同一事件间隔内发生</li>
+<li><p>分词词典</p></li>
+<li><p>文本扫描顺序</p>
+<ul>
+<li>正向扫描</li>
+<li>逆向扫描</li>
+<li>双向扫描</li>
+</ul></li>
+<li><p>匹配原则</p>
+<ul>
+<li><p>最大匹配（MM）</p>
+<p>假定词典中的最长词条所含汉字个数为i，则被处理文本中前i个字符作为匹配字段，查找词典。如果词典中含有这样i个汉字个数的词，则匹配成功，并且切分出这个词；如果找不到就匹配失败，将匹配字段去掉最后一个汉字，继续匹配，直到匹配成功为止。统计方法表明，错误率为1/169。</p></li>
+<li><p>最小匹配（逆向最大匹配，RMM）。与MM方法相同，只不过是从文本末尾开始，匹配失败就去掉前一个汉字。错误率为1/245。</p></li>
+<li><p>逐词匹配。将词典中的词由长到短逐词遍历，直到文本切分完成。</p></li>
+<li><p>最佳匹配（OM）</p>
+<p>在词典中按词频的大小顺序排列词条，缩短检索时间，降低时间复杂度，加快分词速度。不算是一种分词方法，只是一种对分词词典的一种组织方式。OM
+法的分词词典每条词的前面必须有指明长度的数据项，所以其空间复杂度有所增加，对提高分词精度没有影响，分词处理的时间复杂度有所降低。</p></li>
+</ul></li>
 </ul>
-<h2 id="弱引用">弱引用</h2>
-<p>推荐阅读</p>
-<p><a
-href="http://blog.soliloquize.org/2016/01/21/Python%E5%BC%B1%E5%BC%95%E7%94%A8%E7%9A%84%E4%BD%BF%E7%94%A8%E4%B8%8E%E6%B3%A8%E6%84%8F%E4%BA%8B%E9%A1%B9/">Python弱引用的使用与注意事项</a></p>
-<p>弱引用与强引用相对，是指不能确保其引用的对象不会被垃圾回收器回收的引用。若一个对象若只被弱引用所引用，则被认为是不可访问（或弱可访问）的，病因此可能在任何时刻被回收。主要作用就是减少循环引用，减少内存中不必要的对象存在的数量。</p>
-<p>Python用
-<code>weakref</code>模块创建对象的弱引用，当引用计数为0或只存在对象的弱引用时将回收这个对象。</p>
-<p>内存管理时使用，cache编程，垃圾回收(GC)</p>
-<h2 id="while-1与while-true">while 1与while True</h2>
+<h3 id="基于统计的分词方法">基于统计的分词方法</h3>
+<p>基本思想：词是稳定的组合。</p>
+<p>上下文中，相邻的字同时出现的次数越多，就越有可能构成一个词。</p>
+<p>字与字相邻出现的概率或<a
+href="#frequency">频率</a>反应出词的可信度。</p>
+<p>无字典分词：统计文本中相邻字的组合的<a
+href="#frequency">频度</a>，计算它们之间的互现信息。</p>
+<p>该方法所应用的主要的统计模型有：N
+元文法模型（N-gram）、隐马尔可夫模型（Hiden Markov
+Model，HMM）、最大熵模型（ME）、条件随机场模型（Conditional Random
+Fields，CRF）等。</p>
+<p>在实际应用中此类分词算法一般是将其与基于词典的分词方法结合起来，既发挥匹配分词切分速度快、效率高的特点，又利用了无词典分词结合上下文识别生词、自动消除歧义的优点。</p>
+<h3 id="基于语义的分词方法">基于语义的分词方法</h3>
+<p>语义分析。对自然语言自身的语言信息进行更多的处理。</p>
+<ul>
+<li><p>扩充转移网络法</p>
 <p><a
-href="http://www.pythoner.com/356.html">Python天坑系列（一）：while
-1比while True更快？</a></p>
-<p>在python2中while 1会比while True更快</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">while_one</span>():</span><br><span class="line">    i = <span class="number">0</span></span><br><span class="line">    <span class="keyword">while</span> <span class="number">1</span>:</span><br><span class="line">        i += <span class="number">1</span></span><br><span class="line">        <span class="keyword">if</span> i == <span class="number">10000000</span>:</span><br><span class="line">            <span class="keyword">break</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">while_true</span>():</span><br><span class="line">    i = <span class="number">0</span></span><br><span class="line">    <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">        i += <span class="number">1</span></span><br><span class="line">        <span class="keyword">if</span> i == <span class="number">10000000</span>:</span><br><span class="line">            <span class="keyword">break</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    w1_time, wt_time = <span class="number">0</span>, <span class="number">0</span></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">        w1 = timeit.timeit(while_one, <span class="string">&quot;from __main__ import while_one&quot;</span>, number=<span class="number">3</span>)</span><br><span class="line">        w1_time += w1</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">        wt = timeit.timeit(while_true, <span class="string">&quot;from __main__ import while_true&quot;</span>, number=<span class="number">3</span>)</span><br><span class="line">        wt_time += wt</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;while one: %s\nwhile_true: %s&quot;</span> % (w1_time, wt_time))</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2的输出</span></span><br><span class="line"><span class="keyword">while</span> one: 9.27971386909</span><br><span class="line">while_true: 13.5762348175</span><br><span class="line"><span class="comment"># python3的输出</span></span><br><span class="line"><span class="keyword">while</span> one: 14.19957690299998</span><br><span class="line">while_true: 13.785004297995329</span><br></pre></td></tr></table></figure>
-<p>原因：python2中True/False不是关键字，python3添加了True/False做为关键字。每次在循环的时候，True/False需要被检查，所以while
-1会更快</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Python <span class="number">2.7</span><span class="number">.14</span> (default, Sep <span class="number">23</span> <span class="number">2017</span>, <span class="number">22</span>:06:<span class="number">14</span>)</span><br><span class="line">[GCC <span class="number">7.2</span><span class="number">.0</span>] on linux2</span><br><span class="line"><span class="type">Type</span> <span class="string">&quot;help&quot;</span>, <span class="string">&quot;copyright&quot;</span>, <span class="string">&quot;credits&quot;</span> <span class="keyword">or</span> <span class="string">&quot;license&quot;</span> <span class="keyword">for</span> more information.</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span><span class="keyword">import</span> keyword</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>keyword.kwlist</span><br><span class="line">[<span class="string">&#x27;and&#x27;</span>, <span class="string">&#x27;as&#x27;</span>, <span class="string">&#x27;assert&#x27;</span>, <span class="string">&#x27;break&#x27;</span>, <span class="string">&#x27;class&#x27;</span>, <span class="string">&#x27;continue&#x27;</span>, <span class="string">&#x27;def&#x27;</span>, <span class="string">&#x27;del&#x27;</span>, <span class="string">&#x27;elif&#x27;</span>, <span class="string">&#x27;else&#x27;</span>, <span class="string">&#x27;except&#x27;</span>, <span class="string">&#x27;exec&#x27;</span>, <span class="string">&#x27;finally&#x27;</span>, <span class="string">&#x27;for&#x27;</span>, <span class="string">&#x27;from&#x27;</span>, <span class="string">&#x27;global&#x27;</span>, <span class="string">&#x27;if&#x27;</span>, <span class="string">&#x27;import&#x27;</span>, <span class="string">&#x27;in&#x27;</span>, <span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;lambda&#x27;</span>, <span class="string">&#x27;not&#x27;</span>, <span class="string">&#x27;or&#x27;</span>, <span class="string">&#x27;pass&#x27;</span>, <span class="string">&#x27;print&#x27;</span>, <span class="string">&#x27;raise&#x27;</span>, <span class="string">&#x27;return&#x27;</span>, <span class="string">&#x27;try&#x27;</span>, <span class="string">&#x27;while&#x27;</span>, <span class="string">&#x27;with&#x27;</span>, <span class="string">&#x27;yield&#x27;</span>]</span><br><span class="line"></span><br><span class="line">Python <span class="number">3.6</span><span class="number">.3</span> (default, Oct  <span class="number">3</span> <span class="number">2017</span>, <span class="number">21</span>:<span class="number">45</span>:<span class="number">48</span>)</span><br><span class="line">[GCC <span class="number">7.2</span><span class="number">.0</span>] on linux</span><br><span class="line"><span class="type">Type</span> <span class="string">&quot;help&quot;</span>, <span class="string">&quot;copyright&quot;</span>, <span class="string">&quot;credits&quot;</span> <span class="keyword">or</span> <span class="string">&quot;license&quot;</span> <span class="keyword">for</span> more information.</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span><span class="keyword">import</span> keyword</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>keyword.kwlist</span><br><span class="line">[<span class="string">&#x27;False&#x27;</span>, <span class="string">&#x27;None&#x27;</span>, <span class="string">&#x27;True&#x27;</span>, <span class="string">&#x27;and&#x27;</span>, <span class="string">&#x27;as&#x27;</span>, <span class="string">&#x27;assert&#x27;</span>, <span class="string">&#x27;break&#x27;</span>, <span class="string">&#x27;class&#x27;</span>, <span class="string">&#x27;continue&#x27;</span>, <span class="string">&#x27;def&#x27;</span>, <span class="string">&#x27;del&#x27;</span>, <span class="string">&#x27;elif&#x27;</span>, <span class="string">&#x27;else&#x27;</span>, <span class="string">&#x27;except&#x27;</span>, <span class="string">&#x27;finally&#x27;</span>, <span class="string">&#x27;for&#x27;</span>, <span class="string">&#x27;from&#x27;</span>, <span class="string">&#x27;global&#x27;</span>, <span class="string">&#x27;if&#x27;</span>, <span class="string">&#x27;import&#x27;</span>, <span class="string">&#x27;in&#x27;</span>, <span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;lambda&#x27;</span>, <span class="string">&#x27;nonlocal&#x27;</span>, <span class="string">&#x27;not&#x27;</span>, <span class="string">&#x27;or&#x27;</span>, <span class="string">&#x27;pass&#x27;</span>, <span class="string">&#x27;raise&#x27;</span>, <span class="string">&#x27;return&#x27;</span>, <span class="string">&#x27;try&#x27;</span>, <span class="string">&#x27;while&#x27;</span>, <span class="string">&#x27;with&#x27;</span>, <span class="string">&#x27;yield&#x27;</span>]</span><br></pre></td></tr></table></figure>
-<h2 id="if-x-true-与-if-x">if x == True 与 if x</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">if_x_eq_true</span>():</span><br><span class="line">    x = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">if</span> x == <span class="literal">True</span>:</span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">if_x</span>():</span><br><span class="line">    x = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">if</span> x:</span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    if1 = timeit.timeit(if_x_eq_true, <span class="string">&quot;from __main__ import if_x_eq_true&quot;</span>, number = <span class="number">1000000</span>)</span><br><span class="line">    if2 = timeit.timeit(if_x, <span class="string">&quot;from __main__ import if_x&quot;</span>, number = <span class="number">1000000</span>)</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;if_x_eq_true: %s\nif_x: %s&quot;</span> % (if1, if2))</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2</span></span><br><span class="line">if_x_eq_true: 0.118808984756</span><br><span class="line">if_x: 0.0920550823212</span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line">if_x_eq_true: 0.0951261969985353</span><br><span class="line">if_x: 0.07899275699855934</span><br></pre></td></tr></table></figure>
-<p>不管python2还是python3, <code>if x:</code>从各方面来说都会优于
-<code>if x == True</code>。后者会比前者多出一个检查True值和比较的操作。</p>
-<h2 id="type与object">type与object</h2>
-<p><a href="https://www.zhihu.com/question/38791962">Python 的 type 和
-object 之间是怎么一种关系？</a></p>
-<h2
-id="深刻理解python中的元类metaclass">深刻理解Python中的元类(metaclass)</h2>
+href="https://baike.baidu.com/item/%E6%9C%89%E9%99%90%E7%8A%B6%E6%80%81%E6%9C%BA/2081914?fr=aladdin">有限状态机</a>。有限状态机只能识别正则语言，对有限状态机作的第一次扩充使其具有递归能力，形成递归转移网络
+（RTN）。在RTN
+中，弧线上的标志不仅可以是终极符（语言中的单词）或非终极符（词类），还可以调用另外的子网络名字分非终极符（如字或字串的成词条件）。这样，计算机在
+运行某个子网络时，就可以调用另外的子网络，还可以递归调用。词法扩充转移网络的使用，
+使分词处理和语言理解的句法处理阶段交互成为可能，并且有效地解决了汉语分词的歧义。</p></li>
+<li><p>知识分词语义分析法</p></li>
+<li><p>邻接约束法</p></li>
+<li><p>综合匹配法</p></li>
+<li><p>后缀分词法</p></li>
+<li><p>特征词库法</p></li>
+<li><p>矩阵约束法</p>
+<p>先建立一个语法约束矩阵和一个语义约束矩阵，
+其中元素分别表明具有某词性的词和具有另一词性的词相邻是否符合语法规则，
+属于某语义类的词和属于另一词义类的词相邻是否符合逻辑，机器在切分时以之约束分词结果。</p></li>
+<li><p>语法分析法</p></li>
+<li><p>……</p></li>
+</ul>
+<h3 id="基于理解的分词方法">基于理解的分词方法</h3>
+<p>通过让计算机模拟人对句子的理解，达到识别词的效果。这种分词方法需要使用大量的语言知识和信息。</p>
+<p>基本思想：在分词的同时进行句法、语义分析，利用句法信息和语义信息来处理歧义现象。</p>
+<ul>
+<li>分词子系统</li>
+<li>句法语义子系统</li>
+<li>总控部分</li>
+</ul>
+<p>在总控部分的协调下，分词子系统可以获得有关词、句子等的句法和语义信息来对分词歧义进行判断，即它模拟了人对句子的理解过程。</p>
+<p>主要的方法：</p>
+<ul>
+<li><p>专家系统分词法</p>
+<p>从专家系统角度把分词的知识（包括常识性分词知识与消除歧义切分的启发性知识即歧义切分规则）从实现分词过程的推理机中独立出来，使知识库的维护与推理机的实现互不干扰，从而使知识库易于维护和管理。它还具有发现交集歧义字段和多义组合歧义字段的能力和一定的自学习功能。</p></li>
+<li><p>神经网络分词法</p>
+<p>该方法是模拟人脑并行，分布处理和建立数值计算模型工作的。它将分词知识所分散隐式的方法存入神经网络内部，通过自学习和训练修改内部权值，以达到正确的分词结果，最后给出神经网络自动分词结果，如使用
+LSTM、GRU 等神经网络模型等。</p></li>
+<li><p>神经网络专家系统集成式分词法</p>
+<p>该方法首先启动神经网络进行分词，当神经网络对新出现的词不能给出准确切分时，激活专家系统进行分析判断，依据知识库进行推理，得出初步分析，并启动学习机制对神经网络进行训练。该方法可以较充分发挥神经网络与专家系统二者优势，进一步提高分词效率。</p></li>
+</ul>
+<h2 id="损失函数">损失函数</h2>
 <p><a
-href="http://blog.jobbole.com/21351/">深刻理解Python中的元类(metaclass)</a></p>
+href="https://zhuanlan.zhihu.com/p/35027284?utm_source=com.tencent.tim&amp;utm_medium=social">损失函数整理</a></p>
+<h2 id="补充">补充</h2>
+<p><span id="hanming"></p>
+<h3 id="汉明距离">汉明距离</h3>
+<p>在<a
+href="https://zh.wikipedia.org/wiki/%E4%BF%A1%E6%81%AF%E8%AE%BA">信息论</a>中，两个等长<a
+href="https://zh.wikipedia.org/wiki/%E5%AD%97%E7%AC%A6%E4%B8%B2">字符串</a>之间的<strong>汉明距离</strong>（Hamming
+distance）是两个字符串对应位置的不同字符的个数。换句话说，它就是将一个字符串变换成另外一个字符串所需要替换的字符个数。</p>
+<p><strong>汉明重量</strong>是字符串相对于同样长度的零字符串的汉明距离，也就是说，它是字符串中非零的元素个数：对于<a
+href="https://zh.wikipedia.org/wiki/%E4%BA%8C%E8%BF%9B%E5%88%B6">二进制</a><a
+href="https://zh.wikipedia.org/wiki/%E5%AD%97%E7%AC%A6%E4%B8%B2">字符串</a>来说，就是1的个数，所以11101的汉明重量是4。</p>
+<p></span></p>
+<p><span id="hash"></p>
+<h3 id="hash函数">hash函数</h3>
+<p>hash（散列、杂凑）函数，是将任意长度的数据映射到有限长度的域上。直观解释起来，就是对一串数据m进行杂糅，输出另一段固定长度的数据h，作为这段数据的特征（指纹）。</p>
+<p></span></p>
+<p><span id="frequency"></p>
+<h3 id="频率与频度">频率与频度</h3>
+<p>频率：一定时间（一分钟，一秒，一小时）内，某动作的次数（某事物出现的次数）。</p>
+<p>频度：即时间频度。一个算法执行所耗费的时间。一个算法中的语句执行次数称为语句频度或时间频度。</p>
+<p></span></p>
+<p><span id="Interacting"></p>
+<h3 id="互现信息">互现信息</h3>
+<p>互现信息体现了汉字之间结合关系的紧密程度。当紧密程度高于某一个阈值时，便可以认为此字组可能构成了一个词。</p>
+<p></span></p>
 ]]></content>
       <categories>
-        <category>Python</category>
+        <category>Note</category>
       </categories>
-      <tags>
-        <tag>GIL</tag>
-        <tag>元类</tag>
-      </tags>
   </entry>
   <entry>
-    <title>Python 自动导入包</title>
-    <url>/hexo_blog/2023/12/03/Python/Python%E8%87%AA%E5%8A%A8%E5%AF%BC%E5%85%A5%E5%8C%85/</url>
-    <content><![CDATA[<p>在一个文件夹中，有很多个 Python
-文件，每个文件都有若干重名函数，需要一一导入。这个时候，可以用 Python
-的自动导入包来实现。</p>
+    <title>温度可视化</title>
+    <url>/hexo_blog/2023/12/03/%E6%B8%A9%E5%BA%A6%E5%8F%AF%E8%A7%86%E5%8C%96/</url>
+    <content><![CDATA[<p>时序数据，使用简单的 HTML 对其进行展示。</p>
 <span id="more"></span>
 <h2 id="需求描述">需求描述</h2>
-<p>假设有一个文件夹</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">func/</span><br><span class="line">├── __init__.py</span><br><span class="line">├── func1.py</span><br><span class="line">├── func2.py</span><br><span class="line">├── func3.py</span><br></pre></td></tr></table></figure>
-<p>每个文件都有一个重名函数，比如</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># func1.py</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">foo</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;foo&quot;</span>)</span><br></pre></td></tr></table></figure>
-<p>这不仅导致 <code>import</code> 代码不够简洁，且在新增文件时，还得在
-<code>__init__.py</code> 中添加 <code>import</code>
-语句。甚至在调用处也得增加代码来处理。</p>
+<p>接 <a href="/hexo_blog/2023/11/26/IOT/%E6%A0%91%E8%8E%93%E6%B4%BE/%E6%A0%91%E8%8E%93%E6%B4%BE%E6%8A%98%E8%85%BE%E8%AE%B0%E5%BD%95/" title="树莓派折腾记录">树莓派折腾记录</a> 中"获取小米温湿度计"的数据,
+现在有了若干数据后，希望用一种直观的方式展示出来。</p>
 <h2 id="解决方案">解决方案</h2>
-<p>Python 的自动导入包可以解决这个问题。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> glob</span><br><span class="line"><span class="keyword">import</span> importlib</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取当前文件夹下所有的 .py 文件</span></span><br><span class="line">files = glob.glob(os.path.join(os.path.dirname(__file__), <span class="string">&quot;*.py&quot;</span>))</span><br><span class="line">func_dict = &#123;&#125;</span><br><span class="line"><span class="keyword">for</span> file_ <span class="keyword">in</span> files:</span><br><span class="line">    func_name = os.path.basename(file_)[:-<span class="number">3</span>]</span><br><span class="line">    load_module = importlib.import_module(<span class="string">f&quot;func.<span class="subst">&#123;func_name&#125;</span>&quot;</span>)</span><br><span class="line">    <span class="built_in">globals</span>()[load_module.__name__] = <span class="built_in">getattr</span>(load_module, <span class="string">&quot;foo&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 将其放到一个字典中，以按需处理，</span></span><br><span class="line">    func_dict[func_name] = <span class="built_in">globals</span>()[load_module.__name__]</span><br></pre></td></tr></table></figure>
-<p>这样，不仅不需要在 <code>func/__init__.py</code> 中添加
-<code>import</code>
-语句，也不需要在调用处添加额外的代码。甚至，还要根据需求，从字典中调用对应函数。当然，这需要提前进行约定，比如全部放到<code>func</code>文件夹下，且都用<code>foo</code>这个名字作为函数名。</p>
+<p>不想用复杂的前后端代码，希望用最简单的方式画一个折线图即可。所以解决方案是，用
+Python 处理日志数据，将其保存到 json 中，最后用 HTML
+结合现有的可视化库进行展示。为了有一定的实时性，所以在用 Python
+处理日志时，也需要一个定时任务，以定期更新前端页面的数据。</p>
+<p>一种不太优雅的方案，可以用 Python 的画图库，将其导出为图片，然后用
+HTML 的 img
+标签进行展示。但是这样的话，图片的大小就不好控制了，而且也不太好看。所以还是用
+HTML 的可视化库比较好。</p>
+<figure class="highlight html"><table><tr><td class="code"><pre><span class="line"><span class="meta">&lt;!DOCTYPE <span class="keyword">html</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">img</span> <span class="attr">src</span>=<span class="string">&quot;static/pi_data.png&quot;</span> <span class="attr">alt</span>=<span class="string">&quot;温度曲线&quot;</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre></td></tr></table></figure>
+<h2 id="代码实现">代码实现</h2>
+<p>用 Google 的可视化库 <a
+href="https://www.gstatic.com/charts/loader.js">https://www.gstatic.com/charts/loader.js</a>，下载后将其保存在本地。</p>
+<p>成品展示：<a
+href="https://raspberrypi.tail54f27.ts.net/temp">https://raspberrypi.tail54f27.ts.net/temp</a>。为了节约显示数据，Python
+中只处理过去 24 小时的数据</p>
+<figure class="highlight html"><table><tr><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">script</span> <span class="attr">type</span>=<span class="string">&quot;text/javascript&quot;</span> <span class="attr">src</span>=<span class="string">&quot;static/loader.js&quot;</span>&gt;</span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">script</span> <span class="attr">type</span>=<span class="string">&quot;text/javascript&quot;</span>&gt;</span><span class="language-javascript"></span></span><br><span class="line"><span class="language-javascript">    google.<span class="property">charts</span>.<span class="title function_">load</span>(<span class="string">&#x27;current&#x27;</span>, &#123; <span class="string">&#x27;packages&#x27;</span>: [<span class="string">&#x27;corechart&#x27;</span>] &#125;);</span></span><br><span class="line"><span class="language-javascript">    <span class="comment">/* </span></span></span><br><span class="line"><span class="comment"><span class="language-javascript">        读取 json 文件，约定好几个字段 data, title, left_label, right_label 即可</span></span></span><br><span class="line"><span class="comment"><span class="language-javascript">        其中，data 是一个二维数组，第一列是时间，第二列是温度，第三列是湿度。但第一行是标题 [&quot;横坐标标题&quot;, &quot;第x条线的标题&quot;]。</span></span></span><br><span class="line"><span class="comment"><span class="language-javascript">        left_label 是左边的纵坐标标题，right_label 是右边的纵坐标标题。</span></span></span><br><span class="line"><span class="comment"><span class="language-javascript">    */</span></span></span><br><span class="line"><span class="language-javascript">  </span></span><br><span class="line"><span class="language-javascript">    <span class="title function_">fetch</span>(<span class="string">&#x27;static/pi_data.json&#x27;</span>)</span></span><br><span class="line"><span class="language-javascript">      .<span class="title function_">then</span>(<span class="function"><span class="params">response</span> =&gt;</span> response.<span class="title function_">json</span>())</span></span><br><span class="line"><span class="language-javascript">      .<span class="title function_">then</span>(<span class="function"><span class="params">data</span> =&gt;</span> &#123;</span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> drawChart_callback = <span class="title function_">drawChart</span>(data.<span class="property">data</span>, <span class="string">&quot;chart&quot;</span>, data.<span class="property">title</span>, data.<span class="property">left_label</span>, data.<span class="property">right_label</span>);</span></span><br><span class="line"><span class="language-javascript">        google.<span class="property">charts</span>.<span class="title function_">setOnLoadCallback</span>(drawChart_callback);</span></span><br><span class="line"><span class="language-javascript">      &#125;)</span></span><br><span class="line"><span class="language-javascript">      .<span class="title function_">catch</span>(<span class="function"><span class="params">error</span> =&gt;</span> &#123;</span></span><br><span class="line"><span class="language-javascript">        <span class="variable language_">console</span>.<span class="title function_">error</span>(<span class="string">&#x27;读取JSON文件时出错：&#x27;</span>, error);</span></span><br><span class="line"><span class="language-javascript">      &#125;);</span></span><br><span class="line"><span class="language-javascript">    <span class="keyword">function</span> <span class="title function_">drawChart</span>(<span class="params">data_arr, chart_id, title, left_label, right_label</span>) &#123;</span></span><br><span class="line"><span class="language-javascript">      <span class="keyword">return</span> <span class="keyword">function</span> (<span class="params"></span>) &#123;</span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> data = google.<span class="property">visualization</span>.<span class="title function_">arrayToDataTable</span>(data_arr);</span></span><br><span class="line"><span class="language-javascript"></span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> options = &#123;</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">hAxis</span>: &#123;</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">useFormatFromData</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">viewWindow</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">minValue</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">maxValue</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="attr">viewWindowMode</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">          &#125;,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">legacyScatterChartLabels</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">vAxes</span>: [&#123;</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;useFormatFromData&quot;</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;viewWindow&quot;</span>: &#123; <span class="string">&quot;max&quot;</span>: <span class="literal">null</span>, <span class="string">&quot;min&quot;</span>: <span class="literal">null</span> &#125;,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;minValue&quot;</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;maxValue&quot;</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;title&quot;</span>: left_label,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;logScale&quot;</span>: <span class="literal">false</span>,</span></span><br><span class="line"><span class="language-javascript">          &#125;, &#123;</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;useFormatFromData&quot;</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;viewWindow&quot;</span>: &#123; <span class="string">&quot;max&quot;</span>: <span class="literal">null</span>, <span class="string">&quot;min&quot;</span>: <span class="literal">null</span> &#125;,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;minValue&quot;</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;maxValue&quot;</span>: <span class="literal">null</span>,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;title&quot;</span>: right_label,</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;logScale&quot;</span>: <span class="literal">false</span>,</span></span><br><span class="line"><span class="language-javascript">          &#125;],</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">booleanRole</span>: <span class="string">&quot;certainty&quot;</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">lineWidth</span>: <span class="number">2</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">legend</span>: <span class="string">&quot;top&quot;</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">title</span>: title,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">fontName</span>: <span class="string">&quot;sans-serif&quot;</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">useFirstColumnAsDomain</span>: <span class="literal">false</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">titleTextStyle</span>: &#123; <span class="string">&quot;color&quot;</span>: <span class="string">&quot;#000&quot;</span>, <span class="string">&quot;fontSize&quot;</span>: <span class="number">24</span>, <span class="string">&quot;bold&quot;</span>: <span class="literal">true</span> &#125;,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">series</span>: &#123; <span class="string">&quot;1&quot;</span>: &#123; <span class="string">&quot;targetAxisIndex&quot;</span>: <span class="number">1</span> &#125; &#125;,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">curveType</span>: <span class="string">&quot;&quot;</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">interpolateNulls</span>: <span class="literal">true</span>,</span></span><br><span class="line"><span class="language-javascript">          <span class="attr">series</span>: &#123;</span></span><br><span class="line"><span class="language-javascript">            <span class="string">&quot;2&quot;</span>: &#123;</span></span><br><span class="line"><span class="language-javascript">                <span class="string">&quot;targetAxisIndex&quot;</span>: <span class="number">1</span></span></span><br><span class="line"><span class="language-javascript">            &#125;</span></span><br><span class="line"><span class="language-javascript">          &#125; <span class="comment">/* 这个字段用于控制第几列放在右边？作为调整左右纵坐标位置的参数，2表示用第3列的数据*/</span></span></span><br><span class="line"><span class="language-javascript">        &#125;;</span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> isDefaultVisualization = <span class="literal">true</span>;</span></span><br><span class="line"><span class="language-javascript">        <span class="keyword">var</span> chart = <span class="keyword">new</span> google.<span class="property">visualization</span>.<span class="title class_">LineChart</span>(<span class="variable language_">document</span>.<span class="title function_">getElementById</span>(chart_id));</span></span><br><span class="line"><span class="language-javascript"></span></span><br><span class="line"><span class="language-javascript">        chart.<span class="title function_">draw</span>(data, options, isDefaultVisualization);</span></span><br><span class="line"><span class="language-javascript">      &#125;</span></span><br><span class="line"><span class="language-javascript">    &#125;</span></span><br><span class="line"><span class="language-javascript">  </span><span class="tag">&lt;/<span class="name">script</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">style</span>&gt;</span><span class="language-css"></span></span><br><span class="line"><span class="language-css">    <span class="selector-class">.container</span> &#123;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">display</span>: flex;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">flex-wrap</span>: wrap;</span></span><br><span class="line"><span class="language-css">    &#125;</span></span><br><span class="line"><span class="language-css"></span></span><br><span class="line"><span class="language-css">    <span class="selector-class">.chart</span> &#123;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">flex-basis</span>: <span class="number">50%</span>;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">width</span>: auto;</span></span><br><span class="line"><span class="language-css">      <span class="attribute">height</span>: <span class="number">400px</span>;</span></span><br><span class="line"><span class="language-css">    &#125;</span></span><br><span class="line"><span class="language-css">  </span><span class="tag">&lt;/<span class="name">style</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">&quot;chart&quot;</span> <span class="attr">class</span>=<span class="string">&quot;chart&quot;</span>&gt;</span><span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
-        <category>Python</category>
+        <category>Visualization</category>
       </categories>
       <tags>
-        <tag>importlib</tag>
-      </tags>
+        <tag>树莓派</tag>
+        <tag>Google Charts</tag>
+      </tags>
   </entry>
   <entry>
-    <title>Python装饰器</title>
-    <url>/hexo_blog/2018/01/30/Python/Python%E8%A3%85%E9%A5%B0%E5%99%A8/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Python的装饰器是什么和一些Python内置装饰器的使用</p>
+    <title>记一次微信公众号爬虫的经历（微信文章阅读点赞的获取）</title>
+    <url>/hexo_blog/2017/11/18/%E8%AE%B0%E4%B8%80%E6%AC%A1%E5%BE%AE%E4%BF%A1%E5%85%AC%E4%BC%97%E5%8F%B7%E7%88%AC%E8%99%AB%E7%9A%84%E7%BB%8F%E5%8E%86%EF%BC%88%E5%BE%AE%E4%BF%A1%E6%96%87%E7%AB%A0%E9%98%85%E8%AF%BB%E7%82%B9%E8%B5%9E%E7%9A%84%E8%8E%B7%E5%8F%96%EF%BC%89/</url>
+    <content><![CDATA[<p>如何爬取微信公众号所有历史文章的阅读量和点赞量？该文介绍了两种突破难点的方法：一是使用微信个人订阅号进行爬取，二是在电脑上使用Fiddler进行抓包分析。详细的操作步骤及代码实现都在文中。缺陷是需要手动获取一些关键参数。针对被封禁的问题，文中提供了两种解决方案。</p>
 <span id="more"></span>
-<p>参考文章:</p>
+<p>项目实现地址。查看我的<a
+href="https://github.com/wnma3mz/wechat_articles_spider">Github地址</a></p>
+<p><a href="https://data.wnma3mz.cn/">数据展示（完整指标）</a></p>
 <p><a
-href="https://www.zhihu.com/question/26930016">如何理解Python装饰器？</a></p>
+href="https://data.wnma3mz.cn/demo.html">日更两次，获取公众号的最新文章链接</a>，暂不支持实时获取阅读点赞</p>
+<p>支持下载文章内容为HTML（含图片，经过二次处理可转换为PDF、Word）。欢迎技术交流，微信：wnma3mz，烦请进行相关备注，如hexo_wechat_spider。二维码见文末，样例图：</p>
+<p>目前具体可获取指标（包含但不局限）：阅读数、点赞数（在看）、评论内容及总数、正文内容及图片、是否为头条、是否为原创。
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/20200319174821639.png" /></p>
+<p>项目已经实现，比本文介绍的更加方便简单，可直接调用。</p>
+<p>关于批量关注公众号的问题已解决，见我的另一篇文章:<a
+href="https://blog.csdn.net/wnma3mz/article/details/105448808">自动批量关注微信公众号（非逆向）</a></p>
+<p>不求完美，只求能够用。。。</p>
+<p>截至2019年4月项目可正常运行， 方法已更新。</p>
+<p>3月1号更新：</p>
+<ol type="1">
+<li>获取阅读点赞时，每篇文章间隔10s 一次性可获取500篇文章以上</li>
+<li>从公众号获取永久链接时，间隔3分钟，可以连续获取几小时（网友测试）</li>
+</ol>
+<p>公开已爬取的公众号历史文章的永久链接，数据上传至GitHub，日期均截止commit时间。</p>
 <p><a
-href="https://www.cnblogs.com/cicaday/p/python-decorator.html">详解Python的装饰器</a></p>
-<h2 id="装饰器是什么">装饰器是什么</h2>
-<p>python的装饰器本身就是一个函数，使用它可以在某些场景下简化python代码，增加一些额外功能。我的理解是，python装饰器可以帮助同时多个函数做一样的事情（比如打印日志），而避免了每个函数都需要额外加上一样的代码。</p>
-<p>举个例子，假设我们有两个函数</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
-<p>以上一段代码片段很容易理解，运行结果就是两行输出。那么问题来了，如果这个时候我需要在每个函数里面加上一个输出，比如<code>print("this is func")</code>。很简单，我们想到了第一种解决方案就是同时在两个函数里面加上这么一行。</p>
-<p>但是如果我们需要在很多个函数在上很多个这种重复代码，我们这么做就很浪费时间了，而且容易出错。这个时候我们想到了第二种解决方案。定义一个新的函数，在每次使用函数之前，先调用新函数。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">    func()</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_print(func_one)</span><br><span class="line">    func_print(func_two)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<p>感觉不错，但是如果这样无疑改变了原来的逻辑结构，且每次需要传递函数给<code>func_print</code>。下面就轮到我们的主角登场了——装饰器。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">        <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">    <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one = func_print(func_one)</span><br><span class="line">    func_two = func_print(func_two)</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
-<p>这样也能达到我们的想要的效果，但是比之前的实现方式更难看懂。而且调用起来似乎更复杂了。</p>
-<p>第一个问题：更难看懂</p>
-<p>可能是对<code>*args</code>、<code>**kwargs</code>的参数理解不是很好。简单解释一下就是说,<code>*args</code>可以理解为一个<code>list</code>，接收多个参数。<code>**kwargs</code>可以理解为一个<code>dict</code>，接收多个<code>key</code>和<code>value</code>。接收的参数数量不定，可以没有，也可以很多个。再具体一点可以百度看看。</p>
-<p>第二个问题：调用更复杂</p>
-<p>其实上面的例子其实调用装饰器的一种方式，<code>python</code>有<code>@</code>的这个语法糖，所以一般情况下，我们是像下面一样调用的。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">        <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">    <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
-<p>好的，简单的装饰器调用基本就是这样，难度进阶，如何使用带参数的装饰器。换句话就说，如果我们需要根据每个函数自定义一些东西，如何传递参数给装饰器。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func_name</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">decorator</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">        <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;this is %s&quot;</span> % func_name)</span><br><span class="line">            <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">        <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> decorator</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print(<span class="params"><span class="string">&quot;one&quot;</span></span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print(<span class="params">func_name=<span class="string">&quot;two&quot;</span></span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<p>观察<code>func_print</code>部分代码，与之前的不同在于多加了一个<code>decorator</code>函数封装了<code>wrapper</code>函数。我的理解是这样的：<code>func_print</code>作用是装饰器的名字和接收外部的参数，<code>decorator</code>接收外部的函数，<code>wrapper</code>返回结果给函数。<code>decorator</code>翻译为中文是<strong>装饰</strong>。<code>wrapper</code>翻译过来是<strong>包装</strong>。<code>decorator</code>装饰函数，<code>wrapper</code>包装函数。</p>
-<p>当然，<code>decorator</code>和<code>wrapper</code>这两个函数名字可以自定义，在这里为了好理解所以用了这两个名字</p>
-<h2 id="内置装饰器">内置装饰器</h2>
-<h3 id="property"><code>@property</code></h3>
-<p>如果要了解使用这个装饰器，就需要知道在不用装饰器的情况下，写一个属性。这里使用python自带的一个例子。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">C</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">getx</span>(<span class="params">self</span>): <span class="keyword">return</span> <span class="variable language_">self</span>._x</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">setx</span>(<span class="params">self, value</span>): <span class="variable language_">self</span>._x = value</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">delx</span>(<span class="params">self</span>): <span class="keyword">del</span> <span class="variable language_">self</span>._x</span><br><span class="line">    x = <span class="built_in">property</span>(getx, setx, delx, <span class="string">&quot;I&#x27;m the &#x27;x&#x27; property.&quot;</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<p>上面就是定义一个python属性的标准写法，下面使用<code>@property</code>来定义属性</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">C</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line"><span class="meta">    @property</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="string">&quot;I am the &#x27;x&#x27; property.&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>._x</span><br><span class="line"><span class="meta">    @x.setter</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>._x = value</span><br><span class="line"><span class="meta">    @x.deleter</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>._x</span><br></pre></td></tr></table></figure>
-<p>这样有什么好处呢，原来的定义，属性访问并没有做限制。使用<code>@property</code>可以使<code>x</code>这个属性更加安全，从获取、设置必须与属性名一致。</p>
-<p>Emmmm.....我的理解就是加上<code>@property</code>的话，定义属性会更加安全更加优雅。毕竟是面向对象了，天知道你的用户会对你的属性做一些什么奇奇怪怪的事情。</p>
+href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/%E7%A7%91%E6%8A%80%E7%BE%8E%E5%AD%A6.json">科技美学</a></p>
+<p><a
+href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/%E5%85%B1%E9%9D%92%E5%9B%A2%E4%B8%AD%E5%A4%AE.json">共青团中央</a></p>
+<p><a
+href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/AppSo.json">AppSo</a></p>
+<p><a
+href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/%E5%8D%97%E6%96%B9%E5%91%A8%E6%9C%AB.json">南方周末</a></p>
+<h2 id="需求">需求</h2>
+<p>某某微信公众号历史的所有文章的阅读数和点赞数</p>
+<h2 id="难点">难点</h2>
+<ol type="1">
+<li>微信公众号历史的所有文章（来源？？？）</li>
+<li>每篇文章的阅读量和点赞量（电脑上浏览文章只显示内容，没有阅读量、点赞量、评论……）</li>
+</ol>
+<h3 id="突破难点一">突破难点一</h3>
+<ol type="1">
+<li><a
+href="http://weixin.sogou.com/">搜狗微信搜索</a>，可以搜索微信公众号文章。但是貌似只能显示该公众号最近十篇的文章。放弃……</li>
+<li>利用抓包工具（Fiddler），抓取文章。成本有点大……，且貌似只能抓取原创文章。不符合个人需求，放弃……</li>
+<li>利用微信个人订阅号进行爬取，神奇的操作。</li>
+</ol>
+<h4 id="操作">操作</h4>
+<ol type="1">
+<li>拥有一个微信个人订阅号，附上登陆和注册链接。<a
+href="https://mp.weixin.qq.com/">微信公众平台</a></li>
+<li>好在之前无聊注册过一次，所以就可以直接登陆操作。没有注册的童鞋可以用自己的微信号注册一下，过程十分简单，在此就不赘述了</li>
+<li>登陆之后，点击左侧菜单栏“管理”-“素材管理”。再点击右边的“新建图文素材”</li>
+</ol>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxMzExMzM2.png" /></p>
+<ol start="4" type="1">
+<li>弹出一个新的标签页，在上面的工具栏找到“超链接”并点击 <img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxNDM5MzM1.png" /></li>
+<li>弹出了一个小窗口，选择“查找文章”，输入需要查找的公众号，这里用“科技美学”公众号作为例子
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxNTM4MTk5.png" /></li>
+<li>点击之后，可以弹出该公众号的所有历史文章</li>
+</ol>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxNzA1NzQy.png" /></p>
+<h4 id="代码实现">代码实现</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 目标url</span></span><br><span class="line">url = <span class="string">&quot;https://mp.weixin.qq.com/cgi-bin/appmsg&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用Cookie，跳过登陆操作</span></span><br><span class="line">headers = &#123;</span><br><span class="line">  <span class="string">&quot;Cookie&quot;</span>: yourcookie,</span><br><span class="line">  <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36&quot;</span>,</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">需要提交的data</span></span><br><span class="line"><span class="string">以下个别字段是否一定需要还未验证。</span></span><br><span class="line"><span class="string">注意修改yourtoken,number</span></span><br><span class="line"><span class="string">number表示从第number页开始爬取，为5的倍数，从0开始。如0、5、10……</span></span><br><span class="line"><span class="string">token可以使用Chrome自带的工具进行获取</span></span><br><span class="line"><span class="string">fakeid是公众号独一无二的一个id，等同于后面的__biz</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line">data = &#123;</span><br><span class="line">    <span class="string">&quot;token&quot;</span>: yourtoken,</span><br><span class="line">    <span class="string">&quot;lang&quot;</span>: <span class="string">&quot;zh_CN&quot;</span>,</span><br><span class="line">    <span class="string">&quot;f&quot;</span>: <span class="string">&quot;json&quot;</span>,</span><br><span class="line">    <span class="string">&quot;ajax&quot;</span>: <span class="string">&quot;1&quot;</span>,</span><br><span class="line">    <span class="string">&quot;action&quot;</span>: <span class="string">&quot;list_ex&quot;</span>,</span><br><span class="line">    <span class="string">&quot;begin&quot;</span>: number,</span><br><span class="line">    <span class="string">&quot;count&quot;</span>: <span class="string">&quot;5&quot;</span>,</span><br><span class="line">    <span class="string">&quot;query&quot;</span>: <span class="string">&quot;&quot;</span>,</span><br><span class="line">    <span class="string">&quot;fakeid&quot;</span>: yourfakeid,</span><br><span class="line">    <span class="string">&quot;type&quot;</span>: <span class="string">&quot;9&quot;</span>,</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用get方法进行提交</span></span><br><span class="line">content_json = requests.get(url, headers=headers, params=data).json()</span><br><span class="line"><span class="comment"># 返回了一个json，里面是每一页的数据</span></span><br><span class="line"><span class="keyword">for</span> item <span class="keyword">in</span> content_json[<span class="string">&quot;app_msg_list&quot;</span>]:</span><br><span class="line">    <span class="comment"># 提取每页文章的标题及对应的url</span></span><br><span class="line">    <span class="built_in">print</span>(item[<span class="string">&quot;title&quot;</span>], <span class="string">&quot;url&quot;</span>: item[<span class="string">&quot;link&quot;</span>])</span><br></pre></td></tr></table></figure>
+<p>以上，即可爬取微信公众号的一页数据，如果是爬取所有页的数据，则需要改变number进行爬取。</p>
+<p>注：每次抓取完一页之后，最好设定time.sleep(3)。过快会导致爬取失败</p>
+<h3 id="突破难点二">突破难点二</h3>
+<p>这里我使用的方法是在电脑上登陆微信客户端，进行抓包分析。从客户端看推文可以看到阅读量、点赞量。</p>
+<p>我使用的是Fiddler。Fiddller具体使用就不赘述了。下面直接演示操作</p>
+<h4 id="操作-1">操作</h4>
+<ol type="1">
+<li>打开fiddler开始监控</li>
+<li>登陆微信客户端，浏览该公众号的任意一篇推文</li>
+<li>可以观察到这里的内容显示会有阅读量、点赞量、评论等</li>
+<li>观察fiddler的监控数据，如下图显示</li>
+</ol>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxODMzNzAx.png" />
+5. 其中
+<code>/mp/getappmgsext?...</code>是我们推文内容的url，双击之后，fiddler界面右边出现如下图数据</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/wechat/aHR0cDovL2ltZy5ibG9nLmNzZG4ubmV0LzIwMTcxMTE4MTkxOTA0MDAw.png" />
+6. 上图下侧的json里面的
+<code>read_num</code>、<code>like_num</code>分别是阅读量和点赞量</p>
+<p>####代码实现</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 目标url</span></span><br><span class="line">url = <span class="string">&quot;http://mp.weixin.qq.com/mp/getappmsgext&quot;</span></span><br><span class="line"><span class="comment"># 添加Cookie避免登陆操作，这里的&quot;User-Agent&quot;最好为手机浏览器的标识</span></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;Cookie&quot;</span>: yourcookie,</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat QBCore/3.43.27.400 QQBrowser/9.0.2524.400&quot;</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">data = &#123;</span><br><span class="line">    <span class="string">&quot;is_only_read&quot;</span>: <span class="string">&quot;1&quot;</span>,</span><br><span class="line">    <span class="string">&quot;is_temp_url&quot;</span>: <span class="string">&quot;0&quot;</span>,              </span><br><span class="line">    <span class="string">&quot;appmsg_type&quot;</span>: <span class="string">&quot;9&quot;</span>, <span class="comment"># 新参数，不加入无法获取like_num</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">添加请求参数</span></span><br><span class="line"><span class="string">__biz对应公众号的信息，唯一</span></span><br><span class="line"><span class="string">mid、sn、idx分别对应每篇文章的url的信息，需要从url中进行提取</span></span><br><span class="line"><span class="string">key、appmsg_token从fiddler上复制即可</span></span><br><span class="line"><span class="string">pass_ticket对应的文章的信息，貌似影响不大，也可以直接从fiddler复制</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line">params = &#123;</span><br><span class="line">    <span class="string">&quot;__biz&quot;</span>: your__biz,</span><br><span class="line">    <span class="string">&quot;mid&quot;</span>: article_mid,</span><br><span class="line">    <span class="string">&quot;sn&quot;</span>: article_sn,</span><br><span class="line">    <span class="string">&quot;idx&quot;</span>: article_idx,</span><br><span class="line">    <span class="string">&quot;key&quot;</span>: yourkey,</span><br><span class="line">    <span class="string">&quot;pass_ticket&quot;</span>: pass_ticket,</span><br><span class="line">    <span class="string">&quot;appmsg_token&quot;</span>: yourappmsg_token,</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment"># 使用post方法进行提交</span></span><br><span class="line">content = requests.post(url, headers=headers, data=data, params=params).json()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 由于上面这种方法可能会获取数据失败，可以采取字符串拼接这种方法</span></span><br><span class="line">origin_url = <span class="string">&quot;https://mp.weixin.qq.com/mp/getappmsgext?&quot;</span></span><br><span class="line">appmsgext_url = origin_url + <span class="string">&quot;__biz=&#123;&#125;&amp;mid=&#123;&#125;&amp;sn=&#123;&#125;&amp;idx=&#123;&#125;&amp;appmsg_token=&#123;&#125;&amp;x5=1&quot;</span>.<span class="built_in">format</span>(your__biz, article_mid, article_sn, article_idx, yourappmsg_token)</span><br><span class="line">content = requests.post(appmsgext_url, headers=headers, data=data).json()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取其中的阅读数和点赞数</span></span><br><span class="line"><span class="built_in">print</span>(content[<span class="string">&quot;appmsgstat&quot;</span>][<span class="string">&quot;read_num&quot;</span>], content[<span class="string">&quot;appmsgstat&quot;</span>][<span class="string">&quot;like_num&quot;</span>])</span><br></pre></td></tr></table></figure>
+<p>以上即可获取到一篇文章的阅读量、点赞量。
+<code>Cookie</code>、<code>req_id</code>、<code>pass_ticket</code>、<code>key</code>、<code>appmsg_token</code>、<code>__biz</code>利用fiddler获取
+如果是需要多篇文章，需要更改的参数
+<code>mid</code>、<code>sn</code>、<code>idx</code>
+如果是不同公众号，就需要根据url修改 <code>__biz</code>
+多篇文章的爬取，需要根据之前爬取到的url进行提取关键信息，分别爬取。
+注：每次抓取完一页之后，最好设定time.sleep(3)。过快会导致爬取失败。</p>
+<h2 id="写在最后">写在最后</h2>
+<p>以上就是这次微信爬虫的经历。</p>
+<p>需要掌握的基本技能：</p>
+<ol type="1">
+<li>python爬虫的语法</li>
+<li>Chrome、Fiddler基本使用</li>
+<li>网络爬虫基本常识</li>
+</ol>
+<p>缺陷：</p>
+<ol type="1">
+<li>使用Cookie登陆，未实现自动登陆</li>
+<li>key、appmsg_token一些关键参数需要进行手动获取</li>
+<li>实际运行之后，就算设定了爬取间隙时间，还是会被封禁（获取链接时）。</li>
+</ol>
+<p>说明：</p>
+<ol type="1">
+<li>网上一些说法，key半小时过期，我好像没有遇到。</li>
+<li>代码中若有细节问题，欢迎指出讨论</li>
+</ol>
+<p>Github上已经实现了第1、2点，欢迎回到文章开头看github上的实现过程。</p>
+<p>关于被封禁的问题，已有两个解决方案，均放在github上，在这不做讲解。</p>
+<ol type="1">
+<li>selenium解决方案，提高约十倍抓取量（不确定）。优点：提高抓取量；缺点：速度慢，不一定能完全抓取完整，抓取量不确定。<a
+href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/test/test_seleinum/test_seleinum.py">test_seleinum.py</a></li>
+<li>利用个人微信号的查看公众号历史消息，抓取量在500条以上，具体未测试。优点：抓取量最多的方案；缺点：短时间内（5-10分钟）无法查看历史消息，或者说无法持续抓取；不保证微信号会被封号。<a
+href="https://github.com/wnma3mz/wechat_articles_spider/blob/master/test/test_GetUrls.py">test_GetUrls.py</a></li>
+</ol>
 ]]></content>
       <categories>
-        <category>Python</category>
+        <category>Scrapy</category>
       </categories>
       <tags>
-        <tag>Decorators</tag>
+        <tag>Python</tag>
+        <tag>爬虫</tag>
+        <tag>微信</tag>
       </tags>
   </entry>
   <entry>
-    <title>Python魔术方法</title>
-    <url>/hexo_blog/2018/03/14/Python/Python%E9%AD%94%E6%9C%AF%E6%96%B9%E6%B3%95/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Python的魔术方法</p>
+    <title>论文从零到一</title>
+    <url>/hexo_blog/2019/05/28/%E8%AE%BA%E6%96%87%E4%BB%8E%E9%9B%B6%E5%88%B0%E4%B8%80/</url>
+    <content><![CDATA[<p>如何写出一篇论文</p>
 <span id="more"></span>
-<p>参考文章:</p>
-<p><a
-href="http://pycoders-weekly-chinese.readthedocs.io/en/latest/issue6/a-guide-to-pythons-magic-methods.html">Python
-魔术方法指南</a></p>
-<p>魔术方法，顾名思义是一种可以给对象(类)增加魔法的特殊方法，它们的表示方法一般是用双下划线包围(如<code>__init__</code>)</p>
-<h2 id="对象实例化">对象实例化</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> os.path <span class="keyword">import</span> join</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">FileObject</span>:</span><br><span class="line">    <span class="string">&#x27;&#x27;&#x27;给文件对象进行包装从而确认在删除时文件流关闭&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, filepath=<span class="string">&#x27;~&#x27;</span>, filename=<span class="string">&#x27;sample.txt&#x27;</span></span>):</span><br><span class="line">        <span class="comment">#读写模式打开一个文件</span></span><br><span class="line">        <span class="variable language_">self</span>.file = <span class="built_in">open</span>(join(filepath, filename), <span class="string">&#x27;r+&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__del__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="variable language_">self</span>.file.close()</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>.file</span><br></pre></td></tr></table></figure>
+<h2 id="论文从零到一">论文从零到一</h2>
+<p>所谓工欲善其事必先利其器，开头先介绍一些工具与技能（皆非必需）。</p>
+<h3 id="工具与技能">工具与技能</h3>
 <ul>
-<li><p><code>__new__(cls, [...])</code>:
-在一个对象实例化调用的第一个方法，第一个参数是类，剩下的参数传递给<code>__init__</code></p></li>
-<li><p><code>__init__(self, [...])</code>:
-类的初始化方法。当构造函数被调用的时候的任何参数够会传给它</p></li>
-<li><p><code>__del__(self)</code>:
-<code>__new__</code>和<code>__init__</code>为对象的构造器，<code>__del__</code>为对象的析构器。当一个对象进行过垃圾回收时候的行为就是这个方法。当一个对象在删除的时候需要更多的清洁工作，就需要使用这个方法</p></li>
-</ul>
-<h2 id="比较">比较</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Word</span>(<span class="title class_ inherited__">str</span>):</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;存储单词的类，定义比较单词的几种方法&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__new__</span>(<span class="params">cls, word</span>):</span><br><span class="line">        <span class="comment"># 注意我们必须要用到__new__方法，因为str是不可变类型</span></span><br><span class="line">        <span class="comment"># 所以我们必须在创建的时候将它初始化</span></span><br><span class="line">        <span class="keyword">if</span> <span class="string">&#x27; &#x27;</span> <span class="keyword">in</span> word:</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;Value contains spaces. Truncating to first space.&quot;</span>)</span><br><span class="line">            word = word[:word.index(<span class="string">&#x27; &#x27;</span>)] <span class="comment">#单词是第一个空格之前的所有字符</span></span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">str</span>.__new__(cls, word)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__gt__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &gt; <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__lt__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &lt; <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__ge__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &gt;= <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__le__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &lt;= <span class="built_in">len</span>(other)</span><br></pre></td></tr></table></figure>
+<li>LaTex（论文写作工具，当然Word也行），以下是选用工具
 <ul>
-<li><p><code>__cmp__(self, other)</code>:
-用于比较的魔术方法，实现了所有的比较符号。如果<code>self</code> &lt;
-<code>other</code>返回一个负数；如果<code>self</code> ==
-<code>other</code>返回０；如果<code>self</code> &gt;
-<code>other</code>返回正数。一般情况下是分别定义每一个比较符号。如果想实现所有的比较符号，就需要调用这个方法。</p></li>
-<li><p><code>__eq__(self, other)</code>: 等号</p></li>
-<li><p><code>__ne__(self, other)</code>:　不等号</p></li>
-<li><p><code>__lt__(self, other)</code>:　小于</p></li>
-<li><p><code>__gt__(self, other)</code>:　大于</p></li>
-<li><p><code>__le__(self, other)</code>:　小于等于</p></li>
-<li><p><code>__ge__(self, other)</code>:　大于等于</p></li>
+<li><a href="http://www.tablesgenerator.com/latex_tables">Create LaTeX
+tables online</a></li>
+<li><a href="https://www.overleaf.com/?&amp;nocdn=true">Overleaf, Online
+LaTeX Editor</a></li>
+<li><a href="http://www.latextemplates.com/">LaTeX Templates</a></li>
+</ul></li>
+<li>Visio（微软家的画图工具，主要是流程图之类的，当然PPT也行）</li>
+<li>PhotoShop（Emmm，我反正不用）</li>
+<li>Matlab或Python的Matplotlib（绘制展现数据的图表，当然Origin也行）</li>
+<li>Zotero（文献整理工具，当然不用也行）</li>
+<li>SumatraPDF或Adobe Acrobat DC（PDF阅读器）</li>
+<li>知网（论文启蒙网站。。。更多的推荐谷歌学术，Web Of
+Science与Arxiv）</li>
+<li>Sci-Hub或Sci-Hub的telegram bot，同时推荐<a
+href="https://lunwen.im/">https://lunwen.im/</a>（论文下载）</li>
+<li>天若OCR文字识别（OCR文字识别）</li>
+<li>CopyTranslator（复制翻译）</li>
+<li>Mathpix
+Snip（图片转公式，这个需要订阅了，手机端还是免费的，绕过方法见附录）</li>
+<li>1Checker和Grammarly（语法检查工具）</li>
 </ul>
-<h2 id="一元操作符和函数">一元操作符和函数</h2>
+<p>以上除PhotoShop外，其余我在用，当然大部分其实不用或用其他的也行。说白了单凭Word能不能写出好论文，那肯定是可以的。这些只不过是外力罢了，主要还是看自己。</p>
+<h3 id="正文">正文</h3>
+<h4 id="文献调研">文献调研</h4>
+<p>诸如联系老师和确定方向之类的在这里就不谈了，从文献检索开始介绍。如果是一个未接触过的方向，如何从零开始学习呢。这里如果是导师给出一个论文大礼包自然是最好，但我自己的经历告诉我还是自己动手丰衣足食。我自己的方法是，从知网开始学习，当然如果英语足够的话，可以直接跳过此步骤。知网检索相关关键词，筛选出硕士论文与博士论文，下载20篇（一页左右）。这里题外话，关于知网PDF格式下载问题，请使用<a
+href="http://gb.oversea.cnki.net/kns55/default.aspx">海外知网</a>，如果学校没版权的话请另寻他路（方法很多在此不介绍，个人还是应该具备基本的检索能力）。</p>
+<p>为什么选择学位论文呢？因为中国的学位论文是用中文写的，看得懂且质量上有一定的保障。学位论文往往是一个研究生几年内几个工作的总结，故在背景与方法介绍上会更为全面，语言对于普通读者更为友好，专业术语大多都进行了解释。所以在阅读时，首先可以直接从头开始阅读（甚至可以略去摘要），了解清楚一个行业背景和相关方法后，后面阐述作者的工作（文章主体）可以暂时略去不看。在阅读完20篇（甚至不需要20篇）后，基本可以了解清楚该方向的背景与相关知识，到此为止相当于进行了一个方向的科普工作。更进一步，需要对20篇中感兴趣的工作进行更进一步的阅读与了解，此时就需要进行文献整理，提取有效信息。这样可以学习到别人的方法与应用场景，甚至可以尝试去复现，提出自己的观点。</p>
+<p>第一步基本到此结束，但这还远远不够。哪怕是最新的中文文章，可能离国际前沿还是具有一定的差距，且有些方向很新的工作，国内相关论文还比较少。所以，接下来就需要学习英文论文。使用<a
+href="https://scholar.google.cn/">谷歌学术</a>或<a
+href="http://apps.webofknowledge.com/UA_GeneralSearch_input.do?product=UA&amp;search_mode=GeneralSearch&amp;SID=8BsQBbbIKZlBjqwLoEq&amp;preferencesSaved=">Web
+Of
+Science</a>，同样也是搜索关键词（英文），或者根据前面阅读的学位论文查看哪些英文文献经常被引用，可以直接搜索相关文献。除了直接搜索关键词，也可以考虑以下几个点来进行搜索（虽然我自己不怎么用这个方法），已知某个教授在该方向有所建树，故可以直接搜索该教授名字；已知某高校学院在此方向处于国际领先水平，可以直接搜索机构等。在得到相关几篇英文论文后，可以根据引用的论文来回溯更早之前的论文研究，也可以借助如谷歌学术等查看某篇文章被哪些文章引用，可以知道该篇文章的方法最新进展（有可能在某个时间段就被研究人员放弃了）。这样在阅读几十篇英文文章后，就能了解到该方向的最新进展与成果（或者直接筛选时间节点为最近一年或半年的已发表论文）。当然，在一开始阅读英文文献时确实会比较吃力，但是在前期有中文文献阅读的基础上（文章会夹杂着专业术语），这样阅读起来还是比较容易的，把领域内名词吃透后，接下来不过是一些动词形容词而已。学习曲线大概是先难后易的一个过程，一开始需要对某篇（或某几篇）优秀论文进行精读，花费大量时间去翻译研究复现，后续就可以泛读了。关于文章阅读量的问题，贵精不贵多，不一定真的要阅读到很多篇论文才能开始自己的工作。</p>
+<p>下一阶段开始前，建议对阅读的文献进行整理，推荐使用<strong>Zotero</strong>，配合坚果云更香。具体使用方法自行检索。还有Arxiv的话，也可以作为文献检索工具来使用，但是有限定专业人群。</p>
+<h4 id="实验">实验</h4>
+<p>其实关于实验部分可介绍的东西，每个专业甚至每个方向的实验内容都不大一样。所以在这里给出的建议就是有问题及时与老师反馈，跟相关方向的师兄师姐交流。如果是硬件资源问题的话，那我也。。。嗯，关于深度学习的话，可以暂时勉强用Google的Colab。还有诸如网络环境这种，各种工具使用这种，还是需要慢慢踩坑的。这里我有几个人的小Trick可以分享出来。</p>
 <ul>
-<li><code>__pos__(self)</code>：实现正号的特性</li>
-<li><code>__neg__(self)</code>：实现负号的特性</li>
-<li><code>__abs__(self)</code>：abs, 绝对值</li>
-<li><code>__invert__(self)</code>：实现~的特性</li>
-</ul>
-<h2 id="普通算数操作符">普通算数操作符</h2>
+<li>谷歌云盘文件获取直链
 <ul>
-<li><code>__add__(self, other)</code>: 实现加法</li>
-<li><code>__sub__(self, other)</code>: 实现减法</li>
-<li><code>__mul__(self, other)</code>: 实现乘法</li>
-<li><code>__floordiv__(self, other)</code>: 实现整除法</li>
-<li><code>__div__(self, other)</code>: 实现除法</li>
-<li><code>__mod__(self, other)</code>: 实现取模</li>
-<li><code>__pow__(self, other)</code>: 实现指数</li>
-<li><code>__lshift__(self, other)</code>: 实现&lt;&lt;</li>
-<li><code>__rshift__(self, other)</code>: 实现&gt;&gt;</li>
-<li><code>__and__(self, other)</code>: 实现&amp;</li>
-<li><code>__or__(self, other)</code>: 实现|</li>
-<li><code>__xor__(self, other)</code>: 实现^</li>
-</ul>
-<h2 id="其他">其他</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">    <span class="variable language_">self</span>.name = value</span><br><span class="line">    <span class="comment">#每当属性被赋值的时候， ``__setattr__()`` 会被调用，这样就造成了递归调用。</span></span><br><span class="line">    <span class="comment">#这意味这会调用 ``self.__setattr__(&#x27;name&#x27;, value)`` ，每次方法会调用自己。这样会造成程序崩溃。</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">    <span class="variable language_">self</span>.__dict__[name] = value  <span class="comment">#给类中的属性名分配值</span></span><br><span class="line">    <span class="comment">#定制特有属性</span></span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">AccessCounter</span>:</span><br><span class="line">    <span class="string">&#x27;&#x27;&#x27;一个包含计数器的控制权限的类每当值被改变时计数器会加一&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, val</span>):</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="number">0</span>)</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;value&#x27;</span>, val)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">        <span class="keyword">if</span> name == <span class="string">&#x27;value&#x27;</span>:</span><br><span class="line">            <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="variable language_">self</span>.counter + <span class="number">1</span>)</span><br><span class="line">    <span class="comment">#如果你不想让其他属性被访问的话，那么可以抛出 AttributeError(name) 异常</span></span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(name, value)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__delattr__</span>(<span class="params">self, name</span>):</span><br><span class="line">        <span class="keyword">if</span> name == <span class="string">&#x27;value&#x27;</span>:</span><br><span class="line">            <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="variable language_">self</span>.counter + <span class="number">1</span>)</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__delattr__(name)]</span><br></pre></td></tr></table></figure>
-<ul>
-<li><p><code>__str__(self)</code>: 返回值，人类可读</p></li>
-<li><p><code>__repr__(self)</code>: 返回值，机器可读</p></li>
-<li><p><code>__unicode__(self)</code>: 返回unicode()</p></li>
-<li><p><code>__hash__(self)</code>: hash()返回一个整形</p></li>
-<li><p><code>__nonzero__(self)</code>: bool()返回的值</p></li>
-<li><p><code>__getattr__(self, name)</code>:
-当用户试图获取一个不存在（或不建议的属性）的属性时，给出一些警告</p></li>
-<li><p><code>__setattr__(self, name, value)</code>:
-无论属性是否存在，都允许你定义对属性的赋值行为</p></li>
-<li><p><code>__delattr__(self, name)</code>： 删除一个属性</p></li>
-</ul>
-<h2 id="封装附加魔术方法">封装附加魔术方法</h2>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">FunctionalList</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;一个封装了一些附加魔术方法比如 head, tail, init, last, drop, 和take的列表类。</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, values=<span class="literal">None</span></span>):</span><br><span class="line">        <span class="keyword">if</span> values <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            <span class="variable language_">self</span>.values = []</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="variable language_">self</span>.values = values</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__len__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__getitem__</span>(<span class="params">self, key</span>):</span><br><span class="line">        <span class="comment">#如果键的类型或者值无效，列表值将会抛出错误</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[key]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__setitem__</span>(<span class="params">self, key, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>.values[key] = value</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__delitem__</span>(<span class="params">self, key</span>):</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>.values[key]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__iter__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">iter</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__reversed__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">reversed</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">append</span>(<span class="params">self, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>.values.append(value)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">head</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">tail</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[<span class="number">1</span>:]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">init</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="comment">#返回一直到末尾的所有元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[:-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">last</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="comment">#返回末尾元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">drop</span>(<span class="params">self, n</span>):</span><br><span class="line">        <span class="comment">#返回除前n个外的所有元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[n:]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">take</span>(<span class="params">self, n</span>):</span><br><span class="line">        <span class="comment">#返回前n个元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[:n]</span><br></pre></td></tr></table></figure>
-<h2 id="反射">反射</h2>
-<ul>
-<li><code>__instancecheck__(self, instance)</code>:
-检查一个实例是否为定义类的实例</li>
-<li><code>__subclasscheck__(self, subclass)</code>:
-检查一个类是否为定义类的子类</li>
-</ul>
-<h2 id="更改状态">更改状态</h2>
-<ul>
-<li><code>__call__(self, [...])</code>:
-经常改变状态的时候，调用这个方法是一种直接和优雅的方法</li>
+<li>共享文件，获取文件id</li>
+<li>海外服务器，获取转换脚本<code>wget --no-check-certificate -qO /usr/local/bin/gdlink 'https://moeclub.org/attachment/LinuxShell/gdlink.sh' &amp;&amp; chmod a+x /usr/local/bin/gdlink</code></li>
+<li>运行gdlink，获取直链接。gdlink
+'18v_9Bet11B3ZZ3GqWCOa3yTuY2Lzzaav'。输出可直接下载</li>
+</ul></li>
+<li>海外服务器可以利用Github的学生包薅一下<strong>digitalocean</strong>的羊毛，时长一年左右；微软的貌似也可以，还没试。</li>
+<li>关于内网的问题，如果是局域网还好说，可以使用VSCode的插件编辑代码，传输文件也比较方便。如果不是局域网的话，可以参考我的<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>
+使用ssh进行配置，几行命令就ok了，我感觉比一般的内网穿透方便很多。当然需要一个中转服务器这种。</li>
 </ul>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Entity</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;调用实体来改变实体的位置。&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, size, x, y</span>):</span><br><span class="line">        <span class="variable language_">self</span>.x, <span class="variable language_">self</span>.y = x, y</span><br><span class="line">        <span class="variable language_">self</span>.size = size</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__call__</span>(<span class="params">self, x, y</span>):</span><br><span class="line">        <span class="string">&#x27;&#x27;&#x27;改变实体的位置&#x27;&#x27;&#x27;</span></span><br><span class="line">        <span class="variable language_">self</span>.x, <span class="variable language_">self</span>.y = x, y</span><br></pre></td></tr></table></figure>
-<h2 id="会话管理">会话管理</h2>
-<p>使用with语句来调用</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Closer</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;通过with语句和一个close方法来关闭一个对象的会话管理器&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, obj</span>):</span><br><span class="line">        <span class="variable language_">self</span>.obj = obj</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__enter__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.obj <span class="comment"># bound to target</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__exit__</span>(<span class="params">self, exception_type, exception_val, trace</span>):</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            <span class="variable language_">self</span>.obj.close()</span><br><span class="line">        <span class="keyword">except</span> AttributeError: <span class="comment"># obj isn&#x27;t closable</span></span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&#x27;Not closable.&#x27;</span>)</span><br><span class="line">        <span class="keyword">return</span> <span class="literal">True</span> <span class="comment"># exception handled successfully</span></span><br></pre></td></tr></table></figure>
-<h2 id="slots__"><code>__slots__</code></h2>
-<p>限制class的属性，但不限制class的子类的属性，除非子类也使用<code>__slots__</code></p>
-<p>例子：</p>
-<p>使用<code>__slots__</code>会比不使用，减少40%~50%的内存</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 不使用__slots__</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyClass</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">  <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, name, identifier</span>):</span><br><span class="line">      <span class="variable language_">self</span>.name = name</span><br><span class="line">      <span class="variable language_">self</span>.identifier = identifier</span><br><span class="line">      <span class="variable language_">self</span>.set_up()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用__slots__</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyClass</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">  __slots__ = [<span class="string">&#x27;name&#x27;</span>, <span class="string">&#x27;identifier&#x27;</span>]</span><br><span class="line">  <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, name, identifier</span>):</span><br><span class="line">      <span class="variable language_">self</span>.name = name</span><br><span class="line">      <span class="variable language_">self</span>.identifier = identifier</span><br><span class="line">      <span class="variable language_">self</span>.set_up()</span><br></pre></td></tr></table></figure>
+<h4 id="写作与投稿">写作与投稿</h4>
+<p>写作部分，大部分都可以从网上搜得到。这里给个建议是说，可以先从模仿开始，不是复制粘贴，是模仿其他论文的格式以及图表形式（用的什么图说明的什么问题）。写作工具还是推荐使用LaTex，最省心的其实就两点排版与插入参考文献。当然还有很多其他优点，比如数学公式的书写。这里引用文献的话，就体现出了为什么要整理文献这一步骤，如果借助文献整理工具，更是可以减少很多不必要的时间。如果没有工具的话，就直接使用谷歌学术导出引用格式（具体自行检索）；有工具的话，可以直接利用工具一键导出，无论是word写作还是LaTex写作。</p>
+<p>图片一律推荐使用svg格式，但是由于LaTex不直接支持svg，故需要换成eps格式。绘图工具在前面也进行了介绍，自行学习。通篇文章写完之后，建议是找使用英文为母语的人进行阅读（这一点挺难的对于大多国内学生），所以可以使用语法检查工具（在前面也有介绍）。还有一些词汇的使用，避免某些词高频使用，查阅同义词词典进行替换，也可以提高论文的语言水平。</p>
+<p>投稿的话，第一步就是期刊选择的问题。这个一般可以由老师推荐，如果没有推荐的话，可以使用<a
+href="http://www.letpub.com.cn/index.php?page=journalapp&amp;view=search">lethub</a>进行查找相关方向的期刊并且利用<a
+href="https://www.scimagojr.com/journalrank.php">journalrank</a>全面的查找IF和分区等。这里值得注意的是投稿时间与投稿难度仅做部分参考，可信度有待商榷，更科学地方法是到期刊官网看最近的收录期刊以及收录论文的方向。关于OA的问题，OA，OpenAccess，开放期刊，读者可免费阅读期刊内的论文。所以此类期刊的收录一般更大，审稿速度更快，但与之相对的版面费更高，根据自己的实际期刊慎投。一般来说，期刊提交初稿都比较随意，可以直接提交pdf或docx，可以不提交原始稿件这种。后续要修改的话，比如LaTex官网会有对应的模板文件可以直接引用，十分方便。Word的话不是很清楚，可能编辑会帮忙。关于推荐审稿人的问题，有些期刊在提交时会要求推荐审稿人，这里应该是可以随意的，也不一定真的是推荐审稿人审稿，故这里可以推荐关系好的朋友。。。关于CoverLetter的问题，有的期刊是需要提交CoverLetter，类似于自荐信这种，格式网上也基本找得到。基本内容大概就是论文内容以及个人联系方式，为什么要投该期刊等。</p>
+<p>审稿后，返回的意见也就那几种，具体意思可以查阅网上相关内容。这里仅提一点就是，审稿意见需要客观看待，不是拒了就表示文章不行，需要坚持投稿，根据审稿意见改进后，投其他期刊。关于一稿多投的问题，这种问题还是不要出现，不然会对论文的作者们产生不良影响。</p>
+<p>祝各位论文高效产出，投稿必中！</p>
+<p>P.S. 如还有相关疑惑，欢迎留言or邮件wnma3mz@gmail.com！</p>
+<h3 id="附录">附录</h3>
+<h4 id="latex">LaTex</h4>
+<p><strong>LaTeX</strong>，是一种基于<a
+href="https://zh.wikipedia.org/wiki/TeX">TeX</a>的<a
+href="https://zh.wikipedia.org/wiki/排版">排版</a>系统，由<a
+href="https://zh.wikipedia.org/wiki/美国">美国</a><a
+href="https://zh.wikipedia.org/wiki/计算机科学">计算机科学</a>家<a
+href="https://zh.wikipedia.org/wiki/莱斯利·兰伯特">莱斯利·兰伯特</a>在20世纪80年代初期开发，利用这种格式系统的处理，即使使用者没有排版和程序设计的知识也可以充分发挥由TeX所提供的强大功能，不必一一亲自去设计或校对，能在几天，甚至几小时内生成很多具有书籍品质的印刷品。对于生成复杂表格和<a
+href="https://zh.wikipedia.org/wiki/数学">数学</a>公式，这一点表现得尤为突出。因此它非常适用于生成高印刷质量的<a
+href="https://zh.wikipedia.org/wiki/科技">科技</a>和数学、物理文档。这个系统同样适用于生成从简单的信件到完整书籍的所有其他种类的文档。（摘自Wikipedia）</p>
+<p>简单总结，LaTex是一种很nb的排版软件。对于数学、物理、计算机等需要大量数学公式的相关专业学生，强烈建议学习。入门书籍请参考：<a
+href="https://github.com/wnma3mz/Reading-Books/blob/master/非技术/latex教程中文.pdf">latex教程中文.pdf</a>与<a
+href="https://github.com/wnma3mz/Reading-Books/blob/master/非技术/一份不太简短的%20LATEX%202ε%20介绍.pdf">一份不太简短的
+LATEX 2ε 介绍.pdf</a></p>
+<p>几大不得不说的好处：</p>
+<ol type="1">
+<li>引用参考文献，只需要一个<code>bib</code>文件，参考文献风格只需要一行命令便可定义</li>
+<li>数学公式的书写</li>
+<li>各种排版可随时切换</li>
+</ol>
+<h4 id="绘图相关">绘图相关</h4>
+<p>在一篇论文，好看的图表和不可缺少的。大部分数据图用 Matplotlib
+即可完成，流程图也用Visio进行绘制（甚至使用PPT），最后可以考虑使用PS进行完善</p>
+<h4 id="zotero">Zotero</h4>
+<p>开源的文献整理工具。对我而言，它的优势在于：</p>
+<ol type="1">
+<li>云同步（配合坚果云，无限量同步）</li>
+<li>自由上传文件并自动查找文献索引</li>
+<li>一键导出参考文献</li>
+<li>创建时间轴</li>
+<li>批量整理文件</li>
+</ol>
+<h4 id="mathpix-snip">Mathpix Snip</h4>
+<p>对于一个长且复杂的数学公式，我是绝对不想自己重新手写一遍的。此工具可以很方便的将图片转为LaTex公式。在2018年，此软件还是免费使用的状态。但目前已需要付费使用，免费次数于我而言远远不够。同类软件我还没找到很好的替代品。故要继续使用有两种方法，直接付费按年订阅或购买API订阅，再二次封装。均需一定的费用。无意发现安卓版本（IOS不清楚）的调用是免费的，故进行抓包获取了请求API的key，再进行二次封装。目前可稳定运行</p>
 ]]></content>
       <categories>
-        <category>Python</category>
+        <category>Life</category>
+      </categories>
+  </entry>
+  <entry>
+    <title>CUDA编程二次入门</title>
+    <url>/hexo_blog/2018/05/18/CUDA/CUDA%E7%BC%96%E7%A8%8B%E4%BA%8C%E6%AC%A1%E5%85%A5%E9%97%A8/</url>
+    <content><![CDATA[<p>本文主要介绍了基本的CUDA程序框架及代码解析，包括如何申请设备指针内存、将数据拷贝到设备上、调用核函数进行计算、线程同步以及将结果拷贝回主机内等。同时也介绍了一些常见问题及解决方案，比如CUDA程序黑屏之后恢复的问题。接
+<a href="/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/" title="记被CUDA折腾死去活来的那十天">记被CUDA折腾死去活来的那十天</a> 。阅读本文前需要有一部分C语言基础。</p>
+<span id="more"></span>
+<h2 id="原始代码">原始代码</h2>
+<p>在VS2015中，新建一个CUDA项目，里面会有一个
+<code>kernel.cu</code>的文件。里面的代码就是官方给出的实例代码。如下所示。</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>,</span><br><span class="line">        c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h2 id="拆解原始代码">拆解原始代码</h2>
+<p>下面对上面的代码逐一进行拆解进行说明。最原始最简单的代码应该是下面这样。在配置正确的情况下，下面的代码应该也是可以正确运行并输出的。</p>
+<p>设备指的是GPU设备</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 引入相关模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 调用核函数，利用gpu进行计算</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="comment">// 获取当前线程id</span></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="comment">// 在同一个线程id下，a数组与b数组元素对应相加</span></span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line">    <span class="comment">// 一般c语言的语法，在这里不做介绍</span></span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请设备指针，每个变量对应一个设备指针。所以这里有三个</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 选择GPU设备，如果只有一个就只需选择0</span></span><br><span class="line">    cudaSetDevice(<span class="number">0</span>);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请设备指针的内存</span></span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将主机上的a、b数组的数据拷贝到设备上。</span></span><br><span class="line">    cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 调用核函数，注意这里调用函数的语法与一般的不同。&lt;&lt;&lt;1, size&gt;&gt;&gt;指的是分配一个线程组，里面有size个线程</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 进行线程同步</span></span><br><span class="line">    cudaThreadSynchronize();</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将结果拷贝回主机里的c变量中</span></span><br><span class="line">    cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 释放设备指针内存</span></span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 输出结果</span></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>, c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 退出线程</span></span><br><span class="line">    cudaThreadExit();</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p><code>kernel.cu</code>里面的代码是基于上面这个代码进行拓展。</p>
+<ol type="1">
+<li>每一次操作之后，进行判断状态
+<code>cudaStatus</code>，如果状态异常就退出，这是一种比较好的编程习惯，当然带来的影响就是可读性略微降低了。</li>
+<li>封装了一个函数，将操作主机的代码与操作设备的代码（比如申请设备指针内存与释放设备指针等）。</li>
+</ol>
+<h2 id="拓展代码">拓展代码</h2>
+<p>当然，我们还可以基于此再次进行拓展。下面介绍几个比较常用的拓展代码。</p>
+<p>待补充。。。。。</p>
+<h2 id="补充说明">补充说明</h2>
+<p>如果在运行cuda代码的时候，显示屏突然黑屏，之后又恢复正常，是因为程序时间运行过长，触发了TDR事件，导致黑屏，显卡运算中断。解决方案如下：</p>
+<ol type="1">
+<li>开始菜单中找到
+<code>Nsight Monitor</code>并打开。（Win10可以直接搜索）</li>
+<li>在任务栏中打开 <code>Nsight Monitor</code>，单击右下角的
+<code>Nsight Monitor options</code>。</li>
+<li>设置 <code>General</code>的 <code>Microsoft Display Driver</code>中
+<code>WDDM TDR Delay</code>中的值，把
+<code>2</code>(默认一般是2)，调大，比如200。</li>
+<li>保存后退出，重启电脑即可。</li>
+</ol>
+<p>详见<a
+href="http://blog.163.com/yuhua_kui/blog/static/9679964420146183211348/">解决CUDA程序的黑屏恢复问题</a></p>
+]]></content>
+      <categories>
+        <category>CUDA</category>
       </categories>
       <tags>
-        <tag>Magic Methods</tag>
+        <tag>GPU</tag>
       </tags>
   </entry>
   <entry>
-    <title>多进程嵌套内存泄漏问题记录</title>
-    <url>/hexo_blog/2023/11/01/Python/%E5%A4%9A%E8%BF%9B%E7%A8%8B%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F%E9%97%AE%E9%A2%98%E8%AE%B0%E5%BD%95/</url>
-    <content><![CDATA[<p>记一次多进程嵌套发现的内存泄漏问题，以及解决方案。</p>
+    <title>CUDA编程入门</title>
+    <url>/hexo_blog/2018/05/13/CUDA/CUDA%E7%BC%96%E7%A8%8B%E5%85%A5%E9%97%A8/</url>
+    <content><![CDATA[<p>CUDA编程入门，接 <a href="/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/" title="记被CUDA折腾死去活来的那十天">记被CUDA折腾死去活来的那十天</a></p>
 <span id="more"></span>
-<h3 id="问题代码">问题代码</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> multiprocessing</span><br><span class="line"><span class="keyword">from</span> concurrent.futures <span class="keyword">import</span> ProcessPoolExecutor</span><br><span class="line"><span class="keyword">import</span> concurrent.futures</span><br><span class="line"><span class="keyword">from</span> typing <span class="keyword">import</span> <span class="type">Any</span></span><br><span class="line"><span class="keyword">import</span> gc</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Foo</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="variable language_">self</span>.executor = ProcessPoolExecutor(max_workers=<span class="number">10</span>, mp_context=multiprocessing.get_context(<span class="string">&quot;spawn&quot;</span>))</span><br><span class="line"></span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">fun</span>(<span class="params">arg</span>):</span><br><span class="line">        <span class="keyword">raise</span> concurrent.futures.process._RemoteTraceback(<span class="string">&quot;error&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">call</span>(<span class="params">self, i</span>):</span><br><span class="line">        x = []</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">            res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line">    <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;-&quot;</span>*<span class="number">20</span>)</span><br><span class="line">        <span class="keyword">return</span> x</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    <span class="keyword">with</span> ProcessPoolExecutor(max_workers=<span class="number">2</span>, mp_context=multiprocessing.get_context(<span class="string">&quot;spawn&quot;</span>)) <span class="keyword">as</span> e:</span><br><span class="line">        futures = e.submit(predict)</span><br><span class="line">        futures.result()</span><br></pre></td></tr></table></figure>
-<p>肉眼观察 乍一看这段代码是没有任何问题的，在 submit 之后，获取 result
-，并且用 try ... except ... 来捕获可能出现的任何情况。尤其是，当不使用
-main 函数中的进程池时，直接普通的使用 predict
-函数，这段代码不会引发任何问题，可以正常的退出。</p>
-<p>但为什么改成多进程时，这段代码就无法正常退出了呢？程序无法正常退出往往是因为程序某些对象没有完全被释放干净，但这里并没有使用特殊的函数却依旧引发了该问题，所以需要借助工具来进行简单的分析。</p>
-<h3 id="分析">分析</h3>
-<p>garbage collector python 的垃圾回收机制主要使用引用计数（reference
-counting）来跟踪和回收垃圾。大部分情况下，Python
-能够在对象结束执行后，引用计数会变为
-0，使得最后能够被回收。这里程序退出不了的原因是某些对象没发自动退出，所以在
-predict 函数中，观察变量 f 和 f.executor 的引用计数</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f.executor))</span><br></pre></td></tr></table></figure>
-<p>程序的输出，对象 f 的计数为 5，而 f.executor 的计数为 2。为什么是
-2，因此在最外层还有一个进程数量为 2 的进程池没有执行完。而 f 等于 4
-而不等于 f.executor 这就不符合预期，比正常数值多 3。这个 3 是因为循环了
-3 次，每次都引发了异常。</p>
-<p>进一步地，观察 f 中什么对象没有被清理。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line">    <span class="built_in">print</span>(gc.get_referrers(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f.executor))</span><br></pre></td></tr></table></figure>
-<p>此时输出会多一行，形如</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">[&lt;frame at 0x10c266240, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;, &lt;frame at 0x10c6b0240, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;, &lt;frame at 0x10c6b0440, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;]</span><br></pre></td></tr></table></figure>
-<p>这表明程序在何处的对象没有被处理，可以定位到函数 call 的 return
-位置。</p>
-<p>为什么？ 在获取 result 时，如果遇到异常，则中间变量 future
-无法被回收。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">result</span>(<span class="params">self, timeout=<span class="literal">None</span></span>):</span><br><span class="line">    <span class="string">&quot;&quot;&quot;Return the result of the call that the future represents.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Args:</span></span><br><span class="line"><span class="string">        timeout: The number of seconds to wait for the result if the future</span></span><br><span class="line"><span class="string">            isn&#x27;t done. If None, then there is no limit on the wait time.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Returns:</span></span><br><span class="line"><span class="string">        The result of the call that the future represents.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Raises:</span></span><br><span class="line"><span class="string">        CancelledError: If the future was cancelled.</span></span><br><span class="line"><span class="string">        TimeoutError: If the future didn&#x27;t finish executing before the given</span></span><br><span class="line"><span class="string">            timeout.</span></span><br><span class="line"><span class="string">        Exception: If the call raised then that exception will be raised.</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        <span class="keyword">with</span> <span class="variable language_">self</span>._condition:</span><br><span class="line">            <span class="keyword">if</span> <span class="variable language_">self</span>._state <span class="keyword">in</span> [CANCELLED, CANCELLED_AND_NOTIFIED]:</span><br><span class="line">                <span class="keyword">raise</span> CancelledError()</span><br><span class="line">            <span class="keyword">elif</span> <span class="variable language_">self</span>._state == FINISHED:</span><br><span class="line">                <span class="keyword">return</span> <span class="variable language_">self</span>.__get_result()</span><br><span class="line"></span><br><span class="line">    <span class="variable language_">self</span>._condition.wait(timeout)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> <span class="variable language_">self</span>._state <span class="keyword">in</span> [CANCELLED, CANCELLED_AND_NOTIFIED]:</span><br><span class="line">                <span class="keyword">raise</span> CancelledError()</span><br><span class="line">            <span class="keyword">elif</span> <span class="variable language_">self</span>._state == FINISHED:</span><br><span class="line">                <span class="keyword">return</span> <span class="variable language_">self</span>.__get_result()</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                <span class="keyword">raise</span> TimeoutError()</span><br><span class="line">    <span class="keyword">finally</span>:</span><br><span class="line">        <span class="comment"># Break a reference cycle with the exception in self._exception</span></span><br><span class="line">        <span class="variable language_">self</span> = <span class="literal">None</span></span><br></pre></td></tr></table></figure>
-<h3 id="解决方案">解决方案</h3>
-<h4 id="消除中间变量-future">消除中间变量 future</h4>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 改为</span></span><br><span class="line">res = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i).result(timeout=<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
-<h4 id="手动-gc">手动 GC</h4>
-<p>在调用完 Foo 之后，使用 gc.collect()</p>
-<h4 id="主动关闭进程池">主动关闭进程池</h4>
-<p>在调用完 Foo 之后，使用 f.executor.shutdown()</p>
-<h4 id="在-return-前-主动删除未释放的对象">在 return 前
-主动删除未释放的对象</h4>
-<p>如下所示，使用 future = None 也可以达到相似效果（让 Python
-自动调用垃圾回收机制）</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">call</span>(<span class="params">self, i</span>):</span><br><span class="line">   x = []</span><br><span class="line">   <span class="keyword">try</span>:</span><br><span class="line">       future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">       res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line">   <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">       <span class="built_in">print</span>(<span class="string">&quot;-&quot;</span>*<span class="number">20</span>)</span><br><span class="line">       <span class="keyword">del</span> future</span><br><span class="line">   <span class="keyword">return</span> x</span><br></pre></td></tr></table></figure>
-<h3 id="q-a">Q &amp; A</h3>
-<p>Q: 能不能在 Foo 中添加 <strong>del</strong> 函数解决这个问题 A: 在
-Python 3.8.16 可以，Python 3.9.6 不行。可以手动增加 <strong>del</strong>
-函数添加 print 函数，观察是否有对应的输出。</p>
-<p>Q: gc 显示为什么是在 return 这，而不是其他地方？ A: frame
-对象有一个属性叫作 f_lineno，它表示当前执行的代码行。当函数执行到 return
-语句时，f_lineno 会更新为 return 语句所在的行号。</p>
-<p>这是因为在函数返回值之后，控制权会返回到调用该函数的代码。此时函数的上下文（包括它的局部变量、执行状态等）会被丢弃，栈帧也会被弹出调用栈。这个过程称为函数的退出或返回。所以，return
-语句所在的行是函数执行的最后一行。</p>
+<p>参考文章：<a
+href="https://blog.csdn.net/augusdi/article/details/12833235">CUDA从入门到精通</a></p>
+<h2 id="注意事项">注意事项</h2>
+<p>远程桌面登陆服务器时，远程终端不能运行CUDA程序。原因是因为远程桌面登陆调用的显卡资源是本地的显卡，运行程序的时候没有远程的显卡，自然会报错。所以可以用远程终端进行登陆，或者远程服务器配置两块显卡，一块用于显示一块用于计算。</p>
+<p>话不多说，直接上代码。</p>
+<h2 id="原始的demo线程并行">原始的Demo（线程并行）</h2>
+<p>目的就是a, b两个数组对应元素进行相加</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 导入cuda相关模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="comment">// c中自带的模块</span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 声明一个函数后面需要调用</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">// __global__前缀表示用于控制设备，还有一个控制设备的前缀是__device__</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 这个函数是运行在GPU上的，称为核函数</span></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> b[arraySize] = &#123; <span class="number">10</span>, <span class="number">20</span>, <span class="number">30</span>, <span class="number">40</span>, <span class="number">50</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// cuda_Error_t指的是cuda错误类型，取值为整数</span></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="comment">// cudaDeviceProp是设备属性结构体</span></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;&#123;1,2,3,4,5&#125; + &#123;10,20,30,40,50&#125; = &#123;%d,%d,%d,%d,%d&#125;\n&quot;</span>,</span><br><span class="line">        c[<span class="number">0</span>], c[<span class="number">1</span>], c[<span class="number">2</span>], c[<span class="number">3</span>], c[<span class="number">4</span>]);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// cudaThreadExit must be called before exiting in order for profiling and</span></span><br><span class="line">    <span class="comment">// tracing tools such as Nsight and Visual Profiler to show complete traces.</span></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 用于使用CUDA并行添加向量的辅助函数</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// GPU设备端的数据指针</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    <span class="comment">// GPU状态指示</span></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 当电脑上有多个显卡支持CUDA时，选择某个GPU来进行运算。0表示设备号，还有1,2,3……</span></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 分配GPU设备端的内存。三个变量，所以需要分配三次。</span></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="comment">// 申请失败就报错</span></span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// 拷贝数据到GPU中，已有两个变量，所以需要拷贝两次</span></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 运行核函数。关键代码</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 同步线程</span></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 将结果拷贝回主机</span></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    <span class="comment">// 释放GPU设备端内存</span></span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>以上是线程并行，CUDA官方初始给的例子。</p>
+<p>启动核函数的调用过程，即<code>addKernel</code>函数。</p>
+<p>&lt;&lt;&lt;&gt;&gt;&gt;表示运行时配置符号，里面1表示只分配一个线程组（又称线程块、Block），size表示每个线程组有size个线程（Thread）。本程序中size根据前面传递参数个数应该为5，所以运行的时候，核函数在5个GPU线程单元上分别运行了一次，总共运行了5次。这5个线程是如何知道自己“身份”的？是靠<code>threadIdx</code>这个内置变量，它是个dim3类型变量，接受&lt;&lt;&lt;&gt;&gt;&gt;中第二个参数，它包含x,y,z
+3维坐标，而我们传入的参数只有一维，所以只有x值是有效的。通过核函数中<code>int i = threadIdx.x;</code>这一句，每个线程可以获得自身的id号，从而找到自己的任务去执行。</p>
+<h2 id="块并行">块并行</h2>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = blockIdx.x;</span><br><span class="line">	c[i] = a[i] + b[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">addKernel&lt;&lt;&lt;<span class="number">1</span>, size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">addKernel&lt;&lt;&lt;size,<span class="number">1</span> &gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br></pre></td></tr></table></figure>
+<p>这里与线程并行的区别在于，由<code>threadIdx</code>改为了<code>blockIdx</code>。<code>&lt;&lt;&lt;1, size&gt;&gt;&gt;</code>改为了<code>&lt;&lt;&lt;size, 1&gt;&gt;&gt;</code>。</p>
+<p>线程并行是细粒度并行，调度效率高；块并行是粗粒度并行，每次调度都要重新分配资源，有时资源只有一份，那么所有线程块都只能排成一队，串行执行。</p>
+<p>有时采用分治法，将一个大问题分解为几个小规模问题，将这些小规模问题分别用一个线程块实现，线程块内可以采用细粒度的线程并行，而块之间为粗粒度并行，这样可以充分利用硬件资源，降低线程并行的计算复杂度。适当分解，降低规模，在一些矩阵乘法、向量内积计算应用中可以得到充分的展示。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/cuda/20130723220559500.png" /></p>
+<p>多个线程块组成一个线程格(Grid)。一维线程-&gt;二维线程块-&gt;三维线程格。</p>
+<h2 id="流并行">流并行</h2>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line">addKernel&lt;&lt;&lt;size, <span class="number">1</span>&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">cudaStream_t stream[<span class="number">5</span>];</span><br><span class="line"><span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">    <span class="comment">// 创建流</span></span><br><span class="line">    cudaStreamCreate(&amp;stream[i]);</span><br><span class="line">&#125;</span><br><span class="line"><span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">	<span class="comment">// 执行流。线程块数(1)，每个线程中线程数(1)，每个block用到的共享内存大小(0)，流对象(当前核函数在哪个流上运行)</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, <span class="number">1</span>, <span class="number">0</span>, stream[i]&gt;&gt;&gt;(dev_c + i, dev_a + i, dev_b + i);</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 控制整个设备的所有流同步</span></span><br><span class="line">cudaDeviceSynchronize();</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">	cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line"><span class="comment">// 改为</span></span><br><span class="line">Error:</span><br><span class="line">	<span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">5</span>;i++)  &#123;</span><br><span class="line">        <span class="comment">//销毁流</span></span><br><span class="line">        cudaStreamDestroy(stream[i]);</span><br><span class="line">    &#125;</span><br><span class="line">	cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br></pre></td></tr></table></figure>
+<p>利用多个网格（线程格）进行流并行。这样实现的并行可以达到比块并行更粗的细粒度。</p>
+<h2 id="线程通信">线程通信</h2>
+<p>这次目的分三个线程做三个任务，1-5数字进行求和，1-5每个数字求平方和，1-5累乘</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 更改addKernel函数内容如下</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a)</span> &#123;</span><br><span class="line">    <span class="comment">// 线程块数：1，块号为0</span></span><br><span class="line">    <span class="comment">// 线程数：5，线程号0~4</span></span><br><span class="line">    <span class="comment">// 共享存储器大小：5个int型变量大小（5 * sizeof(int）)</span></span><br><span class="line"></span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="comment">// 将数据一次读入到SM（Shared Memory）内部</span></span><br><span class="line">    <span class="keyword">extern</span> __shared__ <span class="type">int</span> smem[];</span><br><span class="line">    smem[i] = a[i];</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 0号线程做平方和</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">0</span>) &#123;</span><br><span class="line">        c[<span class="number">0</span>] = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span>(<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">0</span>] += smem[d] * smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 1号线程做累加</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">1</span>) &#123;</span><br><span class="line">        c[<span class="number">1</span>] = <span class="number">0</span>;</span><br><span class="line">        <span class="keyword">for</span>(<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">1</span>] += smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 2号线程做累乘</span></span><br><span class="line">    <span class="keyword">if</span>(i == <span class="number">2</span>) &#123;</span><br><span class="line">        c[<span class="number">2</span>] = <span class="number">1</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> d = <span class="number">0</span>; d &lt; <span class="number">5</span>; d++) &#123;</span><br><span class="line">            c[<span class="number">2</span>] *= smem[d];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 更改main函数的内容</span></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line"> 	<span class="comment">// 跟普通的C语言代码编写差不多</span></span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">5</span>;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> a[arraySize] = &#123; <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span> &#125;;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123; <span class="number">0</span> &#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus = addWithCuda(c, a, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;\t1+2+3+4+5 = %d\n\t1^2+2^2+3^2+4^2+5^2 = %d\n\t1*2*3*4*5 = %d\n\n\n\n\n\n&quot;</span>, c[<span class="number">1</span>], c[<span class="number">0</span>], c[<span class="number">2</span>]);</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 更改addWithCuda内容</span></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a,  <span class="type">size_t</span> size)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 去除c变量相关内容，如申请内存、GPU设备端指针。其余内容差不多</span></span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)&#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 第三个参数为共享内存大小（字节数）</span></span><br><span class="line">    addKernel&lt;&lt;&lt;<span class="number">1</span>, size, size * <span class="title function_">sizeof</span><span class="params">(<span class="type">int</span>)</span>, 0&gt;&gt;&gt;<span class="params">(dev_c, dev_a)</span>;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h2 id="性能剖析">性能剖析</h2>
+<p>测试对比线程并行和块并行的速度</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;cuda_runtime.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&quot;device_launch_parameters.h&quot;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;stdio.h&gt;</span></span></span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span>;</span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel_blk</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="type">int</span> i = blockIdx.x;</span><br><span class="line">    c[i] = a[i]+ b[i];</span><br><span class="line">&#125;</span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">addKernel_thd</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b)</span> &#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    c[i] = a[i]+ b[i];</span><br><span class="line">&#125;</span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span> &#123;</span><br><span class="line">    <span class="type">const</span> <span class="type">int</span> arraySize = <span class="number">1024</span>;</span><br><span class="line">    <span class="type">int</span> a[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line">    <span class="type">int</span> b[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; arraySize;i++) &#123;</span><br><span class="line">        a[i] = i;</span><br><span class="line">        b[i] = arraySize-i;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="type">int</span> c[arraySize] = &#123;<span class="number">0</span>&#125;;</span><br><span class="line"></span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line">    <span class="type">int</span> num = <span class="number">0</span>;</span><br><span class="line">    <span class="comment">// 设备属性结构体</span></span><br><span class="line">    cudaDeviceProp prop;</span><br><span class="line">    <span class="comment">// 获取设备总数</span></span><br><span class="line">    cudaStatus = cudaGetDeviceCount(&amp;num);</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i&lt;num;i++) &#123;</span><br><span class="line">        <span class="comment">// 获取设备属性</span></span><br><span class="line">        cudaGetDeviceProperties(&amp;prop,i);</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = addWithCuda(c, a, b, arraySize);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;addWithCuda failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaThreadExit();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadExit failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">return</span> <span class="number">1</span>;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i&lt;arraySize;i++) &#123;</span><br><span class="line">        <span class="keyword">if</span>(c[i] != (a[i]+b[i])) &#123;</span><br><span class="line">            <span class="built_in">printf</span>(<span class="string">&quot;Error in %d\n&quot;</span>,i);</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">cudaError_t <span class="title function_">addWithCuda</span><span class="params">(<span class="type">int</span> *c, <span class="type">const</span> <span class="type">int</span> *a, <span class="type">const</span> <span class="type">int</span> *b, <span class="type">size_t</span> size)</span> &#123;</span><br><span class="line">    <span class="type">int</span> *dev_a = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_b = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> *dev_c = <span class="number">0</span>;</span><br><span class="line">    cudaError_t cudaStatus;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaSetDevice(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaSetDevice failed!  Do you have a CUDA-capable GPU installed?&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMalloc((<span class="type">void</span>**)&amp;dev_b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>));</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMalloc failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// Copy input vectors from host memory to GPU buffers.</span></span><br><span class="line">    cudaStatus = cudaMemcpy(dev_a, a, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess)</span><br><span class="line">    &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    cudaStatus = cudaMemcpy(dev_b, b, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyHostToDevice);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 计算核函数执行时间</span></span><br><span class="line">    cudaEvent_t start, stop;</span><br><span class="line">    cudaEventCreate(&amp;start);</span><br><span class="line">    cudaEventCreate(&amp;stop);</span><br><span class="line">    cudaEventRecord(start, <span class="number">0</span>);</span><br><span class="line">    <span class="comment">// 由于数据量小，执行1000次扩大执行时间。blk是块并行，thd是线程并行</span></span><br><span class="line">    <span class="keyword">for</span>(<span class="type">int</span> i = <span class="number">0</span>;i &lt; <span class="number">1000</span>;i++) &#123;</span><br><span class="line"><span class="comment">//      addKernel_blk&lt;&lt;&lt;size,1&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span></span><br><span class="line">        addKernel_thd&lt;&lt;&lt;<span class="number">1</span>,size&gt;&gt;&gt;(dev_c, dev_a, dev_b);</span><br><span class="line">    &#125;</span><br><span class="line">    cudaEventRecord(stop,<span class="number">0</span>);</span><br><span class="line">    cudaEventSynchronize(stop);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// stop-start，执行总时间</span></span><br><span class="line">    <span class="type">float</span> tm;</span><br><span class="line">    cudaEventElapsedTime(&amp;tm, start, stop);</span><br><span class="line">    <span class="built_in">printf</span>(<span class="string">&quot;GPU Elapsed time:%.6f ms.\n&quot;</span>, tm);</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="comment">// cudaThreadSynchronize waits for the kernel to finish, and returns</span></span><br><span class="line">    <span class="comment">// any errors encountered during the launch.</span></span><br><span class="line">    cudaStatus = cudaThreadSynchronize();</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaThreadSynchronize returned error code %d after launching addKernel!\n&quot;</span>, cudaStatus);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// Copy output vector from GPU buffer to host memory.</span></span><br><span class="line">    cudaStatus = cudaMemcpy(c, dev_c, size * <span class="keyword">sizeof</span>(<span class="type">int</span>), cudaMemcpyDeviceToHost);</span><br><span class="line">    <span class="keyword">if</span> (cudaStatus != cudaSuccess) &#123;</span><br><span class="line">        <span class="built_in">fprintf</span>(<span class="built_in">stderr</span>, <span class="string">&quot;cudaMemcpy failed!&quot;</span>);</span><br><span class="line">        <span class="keyword">goto</span> Error;</span><br><span class="line">    &#125;</span><br><span class="line">Error:</span><br><span class="line">    cudaFree(dev_c);</span><br><span class="line">    cudaFree(dev_a);</span><br><span class="line">    cudaFree(dev_b);</span><br><span class="line">    <span class="keyword">return</span> cudaStatus;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
-        <category>Python</category>
+        <category>CUDA</category>
       </categories>
       <tags>
-        <tag>多进程</tag>
-        <tag>内存泄漏</tag>
+        <tag>GPU</tag>
       </tags>
   </entry>
   <entry>
-    <title>Leetcode 字符串刷题记录</title>
-    <url>/hexo_blog/2023/08/28/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E5%AD%97%E7%AC%A6%E4%B8%B2/</url>
-    <content><![CDATA[<p>字符串刷题分类</p>
+    <title>CUDA官方文档笔记</title>
+    <url>/hexo_blog/2018/05/18/CUDA/CUDA%E5%AE%98%E6%96%B9%E6%96%87%E6%A1%A3%E7%AC%94%E8%AE%B0/</url>
+    <content><![CDATA[<p>用CUDA实现高效的矩阵运算——从一维到三维向量加法、共享内存矩阵乘法2.0。接
+<a href="/hexo_blog/2018/05/13/CUDA/CUDA%E7%BC%96%E7%A8%8B%E5%85%A5%E9%97%A8/" title="CUDA编程入门">CUDA编程入门</a>。</p>
 <span id="more"></span>
-<h2 id="滑动窗口">滑动窗口</h2>
-<p><a
-href="https://leetcode.cn/problems/longest-substring-without-repeating-characters/">3.
-无重复字符的最长子串</a></p>
-<p>从左向右滑，需要一个数据结构（A）记录已经遍历的字符，还有一个变量（B）记录最大长度</p>
-<p>注：每次不需要从头开始，而是可以从重复那个字符的后一个开始，所以A需要记录一下索引</p>
-<h2 id="kmp">KMP</h2>
-<h2 id="动态规划">动态规划</h2>
-<p><a
-href="https://leetcode.cn/problems/longest-palindromic-substring/">5.
-最长回文子串</a></p>
-<p>两个指针从两头开始，所以需要二维DP，元素是bool判断</p>
-<p><a href="https://leetcode.cn/problems/interleaving-string/">97.
-交错字符串</a></p>
-<p>两个字符串组成新的字符串，二维DP，元素是bool判断</p>
-<p><a href="https://leetcode.cn/problems/word-break/">139.
-单词拆分</a></p>
-<p>单词数组，一个一个单词拼接，组成最后单词，一维DP，元素是bool判断</p>
-<p><a
-href="https://leetcode.cn/problems/longest-palindromic-subsequence/">516.
-最长回文子序列</a></p>
-<p>回文，需要两个指针，二维DP，从i到j组成回文的长度，元素是长度，最小长度为1，相等+2</p>
-<p><a href="https://leetcode.cn/problems/ones-and-zeroes/">474.
-一和零</a></p>
-<p>字符串数组，组成数量限制（两个限制），所以限制需要二个维度，数组作为最外层那个维度，元素是个数，最小为0</p>
 <p><a
-href="https://leetcode.cn/problems/delete-operation-for-two-strings/">583.
-两个字符串的删除操作</a></p>
-<p>两个字符串，两个维度，删除次数，初始情况，删除个数为字符串长度。</p>
-<p><a href="https://leetcode.cn/problems/palindromic-substrings/">647.
-回文子串</a></p>
-<p>回文，二维DP，元素是bool判断，统计True个数</p>
-<p><a
-href="https://leetcode.cn/problems/longest-common-subsequence/">1143.
-最长公共子序列</a></p>
-<p>两个字符串，二维DP，长度</p>
-<p><a href="https://leetcode.cn/problems/jump-game-vii/">1871. 跳跃游戏
-VII</a></p>
-<p>能否到达终点，一维DP，反向思考，能否到达该点，所以元素为bool。+前缀和优化时间</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 回文串</span></span><br><span class="line">dp = [[<span class="literal">False</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)]</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">    dp[i][i] = <span class="literal">True</span></span><br><span class="line">    <span class="comment"># 从i到j是否组成回文串</span></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(n-<span class="number">1</span>)[::-<span class="number">1</span>]:</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(i+<span class="number">1</span>, n):</span><br><span class="line">            <span class="keyword">if</span> s[i] == s[j] <span class="keyword">and</span> (j - i &lt;= <span class="number">2</span> <span class="keyword">or</span> dp[i+<span class="number">1</span>][j-<span class="number">1</span>]):</span><br><span class="line">                dp[i][j] = <span class="literal">True</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h2 id="双指针">双指针</h2>
-<p><a
-href="https://leetcode.cn/problems/longest-repeating-character-replacement/">424.
-替换后的最长重复字符</a></p>
-<p>枚举右端点，如果没超过限制就继续移动右指针，否则移动左指针</p>
-<h2 id="栈">栈</h2>
-<p>PASS</p>
-<h2 id="字典树">字典树</h2>
-<p>PASS</p>
-<h2 id="模拟深搜">模拟/深搜</h2>
-<p>PASS</p>
+href="https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html">官方文档</a></p>
+<p>一维向量加法1.0</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 核函数定义</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">VecAdd</span><span class="params">(<span class="type">float</span>* A, <span class="type">float</span>* B, <span class="type">float</span>* C)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    C[i] = A[i] + B[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    ...</span><br><span class="line">    <span class="comment">// 用N个线程调用核函数，N目前最多为1024</span></span><br><span class="line">    VecAdd&lt;&lt;&lt;<span class="number">1</span>, N&gt;&gt;&gt;(A, B, C);</span><br><span class="line">    ...</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>二维向量加法1.0</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 核函数，二维数组</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatAdd</span><span class="params">(<span class="type">float</span> A[N][N], <span class="type">float</span> B[N][N],</span></span><br><span class="line"><span class="params">                       <span class="type">float</span> C[N][N])</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = threadIdx.x;</span><br><span class="line">    <span class="type">int</span> j = threadIdx.y;</span><br><span class="line">    C[i][j] = A[i][j] + B[i][j];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    ...</span><br><span class="line">    <span class="comment">// 设置调用GPU参数</span></span><br><span class="line">    <span class="comment">// N * N * 1 threads</span></span><br><span class="line">    <span class="comment">// 一个block，里面有N个线程</span></span><br><span class="line">    <span class="type">int</span> numBlocks = <span class="number">1</span>;</span><br><span class="line">    dim3 <span class="title function_">threadsPerBlock</span><span class="params">(N, N)</span>;</span><br><span class="line">    MatAdd&lt;&lt;&lt;numBlocks, threadsPerBlock&gt;&gt;&gt;(A, B, C);</span><br><span class="line">    ...</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>二维向量加法2.0</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">//</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatAdd</span><span class="params">(<span class="type">float</span> A[N][N], <span class="type">float</span> B[N][N],</span></span><br><span class="line"><span class="params"><span class="type">float</span> C[N][N])</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 使用block</span></span><br><span class="line">    <span class="type">int</span> i = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> j = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="keyword">if</span> (i &lt; N &amp;&amp; j &lt; N)</span><br><span class="line">        C[i][j] = A[i][j] + B[i][j];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    ...</span><br><span class="line">    <span class="comment">// 设置线程参数，block参数。线程block一般为16*16</span></span><br><span class="line">    dim3 <span class="title function_">threadsPerBlock</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span>;</span><br><span class="line">    dim3 <span class="title function_">numBlocks</span><span class="params">(N / threadsPerBlock.x, N / threadsPerBlock.y)</span>;</span><br><span class="line">    MatAdd&lt;&lt;&lt;numBlocks, threadsPerBlock&gt;&gt;&gt;(A, B, C);</span><br><span class="line">    ...</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>一维向量2.0</p>
+<p>这里的设备指的是GPU</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 核函数，使用block</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">VecAdd</span><span class="params">(<span class="type">float</span>* A, <span class="type">float</span>* B, <span class="type">float</span>* C, <span class="type">int</span> N)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> i = blockDim.x * blockIdx.x + threadIdx.x;</span><br><span class="line">    <span class="keyword">if</span> (i &lt; N)</span><br><span class="line">        C[i] = A[i] + B[i];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 主函数，调用CPU</span></span><br><span class="line"><span class="type">int</span> <span class="title function_">main</span><span class="params">()</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">int</span> N = ...;</span><br><span class="line">    <span class="type">size_t</span> size = N * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请主机指针内存</span></span><br><span class="line">    <span class="type">float</span>* h_A = (<span class="type">float</span>*)<span class="built_in">malloc</span>(size);</span><br><span class="line">    <span class="type">float</span>* h_B = (<span class="type">float</span>*)<span class="built_in">malloc</span>(size);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 初始化一些输入向量（参数）</span></span><br><span class="line">    ...</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 申请分配设备内存</span></span><br><span class="line">    <span class="type">float</span>* d_A;</span><br><span class="line">    cudaMalloc(&amp;d_A, size);</span><br><span class="line">    <span class="type">float</span>* d_B;</span><br><span class="line">    cudaMalloc(&amp;d_B, size);</span><br><span class="line">    <span class="type">float</span>* d_C;</span><br><span class="line">    cudaMalloc(&amp;d_C, size);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 拷贝主机内存到设备中</span></span><br><span class="line">    cudaMemcpy(d_A, h_A, size, cudaMemcpyHostToDevice);</span><br><span class="line">    cudaMemcpy(d_B, h_B, size, cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 调用内核</span></span><br><span class="line">    <span class="type">int</span> threadsPerBlock = <span class="number">256</span>;</span><br><span class="line">    <span class="type">int</span> blocksPerGrid =</span><br><span class="line">            (N + threadsPerBlock - <span class="number">1</span>) / threadsPerBlock;</span><br><span class="line">    VecAdd&lt;&lt;&lt;blocksPerGrid, threadsPerBlock&gt;&gt;&gt;(d_A, d_B, d_C, N);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 从设备中拷贝的h_C（结果）到主机中</span></span><br><span class="line">    <span class="comment">// h_C（结果）在主机内存中包含结果</span></span><br><span class="line">    cudaMemcpy(h_C, d_C, size, cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 释放设备内存</span></span><br><span class="line">    cudaFree(d_A);</span><br><span class="line">    cudaFree(d_B);</span><br><span class="line">    cudaFree(d_C);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 释放主机内存</span></span><br><span class="line">    ...</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>二维和三维数组分别需要使用 <code>cudaMallocPitch()</code>和
+<code>cudaMalloc3D()</code></p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 二维数组</span></span><br><span class="line"><span class="comment">// Host code</span></span><br><span class="line"><span class="type">int</span> width = <span class="number">64</span>, height = <span class="number">64</span>;</span><br><span class="line"><span class="type">float</span>* devPtr;</span><br><span class="line"><span class="type">size_t</span> pitch;</span><br><span class="line">cudaMallocPitch(&amp;devPtr, &amp;pitch,</span><br><span class="line">                width * <span class="keyword">sizeof</span>(<span class="type">float</span>), height);</span><br><span class="line">MyKernel&lt;&lt;&lt;<span class="number">100</span>, <span class="number">512</span>&gt;&gt;&gt;(devPtr, pitch, width, height);</span><br><span class="line"></span><br><span class="line"><span class="comment">// Device code</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MyKernel</span><span class="params">(<span class="type">float</span>* devPtr,</span></span><br><span class="line"><span class="params">                         <span class="type">size_t</span> pitch, <span class="type">int</span> width, <span class="type">int</span> height)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> r = <span class="number">0</span>; r &lt; height; ++r) &#123;</span><br><span class="line">        <span class="type">float</span>* row = (<span class="type">float</span>*)((<span class="type">char</span>*)devPtr + r * pitch);</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> c = <span class="number">0</span>; c &lt; width; ++c) &#123;</span><br><span class="line">            <span class="type">float</span> element = row[c];</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 三维数组</span></span><br><span class="line"><span class="comment">// Host code</span></span><br><span class="line"><span class="type">int</span> width = <span class="number">64</span>, height = <span class="number">64</span>, depth = <span class="number">64</span>;</span><br><span class="line">cudaExtent extent = make_cudaExtent(width * <span class="keyword">sizeof</span>(<span class="type">float</span>),</span><br><span class="line">                                    height, depth);</span><br><span class="line">cudaPitchedPtr devPitchedPtr;</span><br><span class="line">cudaMalloc3D(&amp;devPitchedPtr, extent);</span><br><span class="line">MyKernel&lt;&lt;&lt;<span class="number">100</span>, <span class="number">512</span>&gt;&gt;&gt;(devPitchedPtr, width, height, depth);</span><br><span class="line"></span><br><span class="line"><span class="comment">// Device code</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MyKernel</span><span class="params">(cudaPitchedPtr devPitchedPtr,</span></span><br><span class="line"><span class="params">                         <span class="type">int</span> width, <span class="type">int</span> height, <span class="type">int</span> depth)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="type">char</span>* devPtr = devPitchedPtr.ptr;</span><br><span class="line">    <span class="type">size_t</span> pitch = devPitchedPtr.pitch;</span><br><span class="line">    <span class="type">size_t</span> slicePitch = pitch * height;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> z = <span class="number">0</span>; z &lt; depth; ++z) &#123;</span><br><span class="line">        <span class="type">char</span>* slice = devPtr + z * slicePitch;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> y = <span class="number">0</span>; y &lt; height; ++y) &#123;</span><br><span class="line">            <span class="type">float</span>* row = (<span class="type">float</span>*)(slice + y * pitch);</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> x = <span class="number">0</span>; x &lt; width; ++x) &#123;</span><br><span class="line">                <span class="type">float</span> element = row[x];</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 二维、三维数组获取全局变量的方法</span></span><br><span class="line">__constant__ <span class="type">float</span> constData[<span class="number">256</span>];</span><br><span class="line"><span class="type">float</span> data[<span class="number">256</span>];</span><br><span class="line">cudaMemcpyToSymbol(constData, data, <span class="keyword">sizeof</span>(data));</span><br><span class="line">cudaMemcpyFromSymbol(data, constData, <span class="keyword">sizeof</span>(data));</span><br><span class="line"></span><br><span class="line">__device__ <span class="type">float</span> devData;</span><br><span class="line"><span class="type">float</span> value = <span class="number">3.14f</span>;</span><br><span class="line">cudaMemcpyToSymbol(devData, &amp;value, <span class="keyword">sizeof</span>(<span class="type">float</span>));</span><br><span class="line"></span><br><span class="line">__device__ <span class="type">float</span>* devPointer;</span><br><span class="line"><span class="type">float</span>* ptr;</span><br><span class="line">cudaMalloc(&amp;ptr, <span class="number">256</span> * <span class="keyword">sizeof</span>(<span class="type">float</span>));</span><br><span class="line">cudaMemcpyToSymbol(devPointer, &amp;ptr, <span class="keyword">sizeof</span>(ptr));</span><br></pre></td></tr></table></figure>
+<p>共享内存——矩阵乘法1.0</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 定义一个矩阵的数据结构</span></span><br><span class="line"><span class="comment">// Matrices are stored in row-major order:</span></span><br><span class="line"><span class="comment">// M(row, col) = *(M.elements + row * M.width + col)</span></span><br><span class="line"><span class="keyword">typedef</span> <span class="class"><span class="keyword">struct</span> &#123;</span></span><br><span class="line">    <span class="type">int</span> width;</span><br><span class="line">    <span class="type">int</span> height;</span><br><span class="line">    <span class="type">float</span>* elements;</span><br><span class="line">&#125; Matrix;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 线程锁大小</span></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> BLOCK_SIZE 16</span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 提前声明矩阵乘法核函数</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(<span class="type">const</span> Matrix, <span class="type">const</span> Matrix, Matrix)</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 矩阵乘法主机代码</span></span><br><span class="line"><span class="comment">// 矩阵维度设为线程锁大小的倍数</span></span><br><span class="line"><span class="type">void</span> <span class="title function_">MatMul</span><span class="params">(<span class="type">const</span> Matrix A, <span class="type">const</span> Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// Load A and B to device memory</span></span><br><span class="line">    Matrix d_A;</span><br><span class="line">    d_A.width = A.width; d_A.height = A.height;</span><br><span class="line">    <span class="type">size_t</span> size = A.width * A.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_A.elements, size);</span><br><span class="line">    cudaMemcpy(d_A.elements, A.elements, size,</span><br><span class="line">               cudaMemcpyHostToDevice);</span><br><span class="line">    Matrix d_B;</span><br><span class="line">    d_B.width = B.width; d_B.height = B.height;</span><br><span class="line">    size = B.width * B.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_B.elements, size);</span><br><span class="line">    cudaMemcpy(d_B.elements, B.elements, size,</span><br><span class="line">               cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Allocate C in device memory</span></span><br><span class="line">    Matrix d_C;</span><br><span class="line">    d_C.width = C.width; d_C.height = C.height;</span><br><span class="line">    size = C.width * C.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_C.elements, size);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Invoke kernel</span></span><br><span class="line">    dim3 <span class="title function_">dimBlock</span><span class="params">(BLOCK_SIZE, BLOCK_SIZE)</span>;</span><br><span class="line"></span><br><span class="line">    dim3 <span class="title function_">dimGrid</span><span class="params">(B.width / dimBlock.x, A.height / dimBlock.y)</span>;</span><br><span class="line">    MatMulKernel&lt;&lt;&lt;dimGrid, dimBlock&gt;&gt;&gt;(d_A, d_B, d_C);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Read C from device memory</span></span><br><span class="line">    cudaMemcpy(C.elements, d_C.elements, size,</span><br><span class="line">               cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Free device memory</span></span><br><span class="line">    cudaFree(d_A.elements);</span><br><span class="line">    cudaFree(d_B.elements);</span><br><span class="line">    cudaFree(d_C.elements);</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiplication kernel called by MatMul()</span></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(Matrix A, Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// Each thread computes one element of C</span></span><br><span class="line">    <span class="comment">// by accumulating results into Cvalue</span></span><br><span class="line">    <span class="type">float</span> Cvalue = <span class="number">0</span>;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> e = <span class="number">0</span>; e &lt; A.width; ++e)</span><br><span class="line">        Cvalue += A.elements[row * A.width + e]</span><br><span class="line">                * B.elements[e * B.width + col];</span><br><span class="line">    C.elements[row * C.width + col] = Cvalue;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>矩阵乘法2.0</p>
+<p>使用 <code>__device__</code>获取矩阵中元素和设定元素</p>
+<figure class="highlight c"><table><tr><td class="code"><pre><span class="line"><span class="comment">// Matrices are stored in row-major order:</span></span><br><span class="line"><span class="comment">// M(row, col) = *(M.elements + row * M.stride + col)</span></span><br><span class="line"><span class="keyword">typedef</span> <span class="class"><span class="keyword">struct</span> &#123;</span></span><br><span class="line">    <span class="type">int</span> width;</span><br><span class="line">    <span class="type">int</span> height;</span><br><span class="line">    <span class="type">int</span> stride;</span><br><span class="line">    <span class="type">float</span>* elements;</span><br><span class="line">&#125; Matrix;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 获取矩阵元素</span></span><br><span class="line">__device__ <span class="type">float</span> <span class="title function_">GetElement</span><span class="params">(<span class="type">const</span> Matrix A, <span class="type">int</span> row, <span class="type">int</span> col)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="keyword">return</span> A.elements[row * A.stride + col];</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 设定矩阵元素</span></span><br><span class="line">__device__ <span class="type">void</span> <span class="title function_">SetElement</span><span class="params">(Matrix A, <span class="type">int</span> row, <span class="type">int</span> col,</span></span><br><span class="line"><span class="params">                           <span class="type">float</span> value)</span></span><br><span class="line">&#123;</span><br><span class="line">    A.elements[row * A.stride + col] = value;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 根据线程锁的大小构建给定矩阵的子矩阵</span></span><br><span class="line"> __device__ Matrix <span class="title function_">GetSubMatrix</span><span class="params">(Matrix A, <span class="type">int</span> row, <span class="type">int</span> col)</span></span><br><span class="line">&#123;</span><br><span class="line">    Matrix Asub;</span><br><span class="line">    Asub.width    = BLOCK_SIZE;</span><br><span class="line">    Asub.height   = BLOCK_SIZE;</span><br><span class="line">    Asub.stride   = A.stride;</span><br><span class="line">    Asub.elements = &amp;A.elements[A.stride * BLOCK_SIZE * row</span><br><span class="line">                                         + BLOCK_SIZE * col];</span><br><span class="line">    <span class="keyword">return</span> Asub;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">define</span> BLOCK_SIZE 16</span></span><br><span class="line"></span><br><span class="line">__global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(<span class="type">const</span> Matrix, <span class="type">const</span> Matrix, Matrix)</span>;</span><br><span class="line"></span><br><span class="line"><span class="type">void</span> <span class="title function_">MatMul</span><span class="params">(<span class="type">const</span> Matrix A, <span class="type">const</span> Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line"></span><br><span class="line">    Matrix d_A;</span><br><span class="line">    d_A.width = d_A.stride = A.width; d_A.height = A.height;</span><br><span class="line">    <span class="type">size_t</span> size = A.width * A.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_A.elements, size);</span><br><span class="line">    cudaMemcpy(d_A.elements, A.elements, size,</span><br><span class="line">               cudaMemcpyHostToDevice);</span><br><span class="line">    Matrix d_B;</span><br><span class="line">    d_B.width = d_B.stride = B.width; d_B.height = B.height;</span><br><span class="line">    size = B.width * B.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_B.elements, size);</span><br><span class="line">    cudaMemcpy(d_B.elements, B.elements, size,</span><br><span class="line">    cudaMemcpyHostToDevice);</span><br><span class="line"></span><br><span class="line">    Matrix d_C;</span><br><span class="line">    d_C.width = d_C.stride = C.width; d_C.height = C.height;</span><br><span class="line">    size = C.width * C.height * <span class="keyword">sizeof</span>(<span class="type">float</span>);</span><br><span class="line">    cudaMalloc(&amp;d_C.elements, size);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Invoke kernel</span></span><br><span class="line">    dim3 <span class="title function_">dimBlock</span><span class="params">(BLOCK_SIZE, BLOCK_SIZE)</span>;</span><br><span class="line">    dim3 <span class="title function_">dimGrid</span><span class="params">(B.width / dimBlock.x, A.height / dimBlock.y)</span>;</span><br><span class="line">    MatMulKernel&lt;&lt;&lt;dimGrid, dimBlock&gt;&gt;&gt;(d_A, d_B, d_C);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Read C from device memory</span></span><br><span class="line">    cudaMemcpy(C.elements, d_C.elements, size,</span><br><span class="line">               cudaMemcpyDeviceToHost);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Free device memory</span></span><br><span class="line">    cudaFree(d_A.elements);</span><br><span class="line">    cudaFree(d_B.elements);</span><br><span class="line">    cudaFree(d_C.elements);</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiplication kernel called by MatMul()</span></span><br><span class="line"> __global__ <span class="type">void</span> <span class="title function_">MatMulKernel</span><span class="params">(Matrix A, Matrix B, Matrix C)</span></span><br><span class="line">&#123;</span><br><span class="line">    <span class="comment">// 定义块的行与列</span></span><br><span class="line">    <span class="type">int</span> blockRow = blockIdx.y;</span><br><span class="line">    <span class="type">int</span> blockCol = blockIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 每个线程块计算每一个子矩阵</span></span><br><span class="line">    Matrix Csub = GetSubMatrix(C, blockRow, blockCol);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 每个线程计算每个矩阵的元素，结果保留在Cvalue中</span></span><br><span class="line">    <span class="type">float</span> Cvalue = <span class="number">0</span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 定义Csub中线程的行列</span></span><br><span class="line">    <span class="type">int</span> row = threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 对矩阵A、B的所有子矩阵进行循环</span></span><br><span class="line">    <span class="comment">// 计算Csub，把每一对子矩阵Csub循环，之后累加求和</span></span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> m = <span class="number">0</span>; m &lt; (A.width / BLOCK_SIZE); ++m) &#123;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 获取矩阵A、B的子矩阵</span></span><br><span class="line">        Matrix Asub = GetSubMatrix(A, blockRow, m);</span><br><span class="line">        Matrix Bsub = GetSubMatrix(B, m, blockCol);</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 存储A、B矩阵的子矩阵Asub和Bsub用于共享</span></span><br><span class="line">        __shared__ <span class="type">float</span> As[BLOCK_SIZE][BLOCK_SIZE];</span><br><span class="line">        __shared__ <span class="type">float</span> Bs[BLOCK_SIZE][BLOCK_SIZE];</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 从设备中载入Asub与Bsub到共享内存中</span></span><br><span class="line">        <span class="comment">// 每个线程加载每个子矩阵的一个元素</span></span><br><span class="line">        As[row][col] = GetElement(Asub, row, col);</span><br><span class="line">        Bs[row][col] = GetElement(Bsub, row, col);</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 在开始计算前同步以确保子矩阵被加载</span></span><br><span class="line">        __syncthreads();</span><br><span class="line">        <span class="comment">// 同时计算A、B子矩阵</span></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> e = <span class="number">0</span>; e &lt; BLOCK_SIZE; ++e)</span><br><span class="line">            Cvalue += As[row][e] * Bs[e][col];</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 同步以确保A、B子矩阵在下次迭代前计算完成</span></span><br><span class="line">        __syncthreads();</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 写入C子矩阵到设备内存中，每个线程写入一个元素</span></span><br><span class="line">    SetElement(Csub, row, col, Cvalue);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
-        <category>Algorithm</category>
+        <category>CUDA</category>
       </categories>
       <tags>
-        <tag>字符串</tag>
+        <tag>GPU</tag>
       </tags>
   </entry>
   <entry>
-    <title>海量数据处理问题（HashMap+分治法、位图法）</title>
-    <url>/hexo_blog/2018/04/05/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E6%B5%B7%E9%87%8F%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E9%97%AE%E9%A2%98%EF%BC%88HashMap+%E5%88%86%E6%B2%BB%E6%B3%95%E3%80%81%E4%BD%8D%E5%9B%BE%E6%B3%95%EF%BC%89/</url>
-    <content><![CDATA[<p>从海量数据中找出不重复的数和出现次数最多的数的解决方案，分别采用了分治法+HashMap和位图法Bitmap两种方法。其中，位图法在大数据量下处理速度更快，内存占用更小。而对于其他类似问题，可以尝试使用分治法加最小堆/最大堆的方式处理。</p>
+    <title>记被CUDA折腾死去活来的那十天</title>
+    <url>/hexo_blog/2018/05/05/CUDA/%E8%AE%B0%E8%A2%ABCUDA%E6%8A%98%E8%85%BE%E6%AD%BB%E5%8E%BB%E6%B4%BB%E6%9D%A5%E7%9A%84%E9%82%A3%E5%8D%81%E5%A4%A9/</url>
+    <content><![CDATA[<p>谨以此文章纪念大二的一次折腾CUDA环境搭建经历。</p>
 <span id="more"></span>
-<p>原文链接：<a
-href="https://juejin.im/post/5aa0ee9f518825557c010bc0">经典面试问题: Top
-K 之 ---- 海量数据找出现次数最多或，不重复的。</a></p>
-<p>在数据处理中，较为棘手的有这么一种问题，海量数据如何进行处理。一旦数据的数量足够大，我们就不能使用常规的思路来解决。典型例子就是排序算法，在数据不大（不超过内存）的情况下，可以直接使用排序算法，但是当数据量很大时，我们就需要使用如多路归并此类的算法。</p>
-<p>总结问题：</p>
+<p>为了提高矩阵运算的速度和不浪费实验室的GPU，所以开始折腾CUDA环境，入门GPU
+Coder。感谢@小岳岳 全程的帮助，学习了很多硬件知识；感谢@浪浪
+学长的指导，少走了一些弯路。本文会尽量从零开始介绍硬件知识过渡到系统和软件环境的搭建。</p>
+<p>贴上学长的博客<a href="http://amourll.cn/2018/03/27/gpu-coder/">GPU
+Coder初体验</a></p>
+<h2 id="基础介绍">基础介绍</h2>
+<p>首先祭出显卡天梯图，时间截止于博客编辑完的日期（2018年4月）。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/cuda/5af851eca8de0.jpg" /></p>
+<p>关于显卡的一些基本知识可以看知乎的这个贴：<a
+href="https://zhuanlan.zhihu.com/p/33743591">知识扫盲I给大众的电脑显卡选购指南</a></p>
+<p>这里再简单补充一些基础知识，GPU（Graphic Process Unit
+）其实是显卡的核心部件，分为AMD和Nvidia两种。一般情况下，做科学计算或者深度学习都是用Nvidia显卡，当然用AMD的也有在这里我用的是Nvidia所以就不介绍如何搭建AMD的编程环境了。</p>
+<h3 id="关于gpu编程">关于GPU编程</h3>
+<p>所谓GPU编程，我的理解是用代码操作GPU进行运算。通常来说，我们写代码用软件都是调用CPU来工作的，显卡在常规的认知中（某某年以前）都是用来作为图像渲染、玩游戏的。随着时代的发展，关于图像处理的数据越来越多，交给CPU来计算就显得过慢，所以人们开始用起了GPU，GPU原本发明的用途就是为了处理图像。所以如果数据是关于图像处理的，交给GPU来计算远远好于CPU的。</p>
+<h3 id="关于并行计算">关于并行计算</h3>
+<p>GPU并行：CUDA，OpenCL（都比较麻烦，建议CUDA）
+CPU并行：MPI，OpenMP。MPI比较麻烦，OpenMP容易实现</p>
+<p>OpenAcc：OpenMP和CUDA的混合</p>
+<p>这里使用的方案是CUDA（读作：库达）。</p>
+<h3 id="实验室的显卡介绍">实验室的显卡介绍</h3>
+<p>Tesla C2050、两块Titans
+X、一块2010/2011年的AMD卡（具体忘了，反正只用来做图像显示）</p>
+<p>配置情况：特斯拉+AMD，AMD用做显示，特斯拉用来计算。至于为什么不上泰坦，是因为要预备留着后面买的新服务器，这台是之前（2010/2011）的服务器。简单说说特斯拉显卡，特斯拉显示是英伟达专门生产用作科学计算的，一般情况下特斯拉显卡的价格和计算性能都可以碾压其他类型的显卡，而且支持双精度。不过在这的话，泰坦还是可以吊打这块特斯拉的，唯一的缺点大概泰坦是不支持双精度吧。</p>
+<h2 id="正文">正文</h2>
+<h3 id="ubuntu-tesla-amd">Ubuntu + Tesla + AMD</h3>
+<h4 id="结果">结果</h4>
+<p>安装失败</p>
+<h4 id="遇到问题及解决方案">遇到问题及解决方案</h4>
 <ol type="1">
-<li>内存一次性不能放下所有的数</li>
-<li>机器CPU核数不够</li>
-<li>……</li>
+<li>特斯拉不能作为图像显示。解决方案：换AMD显卡做显示。。。</li>
+<li>Ubuntu16.04之后不支持AMD显卡驱动。解决方案：老版本的Ubuntu或者搜索关键词<code>amdgpu-pro</code>、<code>Ubuntu</code></li>
+<li>安装CUDA是否还需要安装英伟达的驱动。Windows下是不需要的，Ubuntu不知道，很多博客中都有安装英伟达驱动这一步骤。</li>
+<li>安装完英伟达驱动之后，出现循环登陆或者不能显示图形界面的情况。解决方案：卸载英伟达驱动，重启。</li>
+<li>英伟达（特斯拉）和AMD双显卡的问题。这种情况网上搜出来的结果比较少，大多都是核显或集显+英伟达显卡。所以在尝试了N次之后，放弃了。</li>
 </ol>
-<h2 id="不重复数">不重复数</h2>
-<p>在2.5亿个正整数中找出不重复的数。</p>
-<h3 id="分治法hashmap">分治法+HashMap</h3>
-<h4 id="说明">说明</h4>
-<p>比如将这些数据平均分为100个批次，每个批次就有250万数。每个批次用循环进行遍历，存入
-<code>HashMap&lt;a, b&gt;</code>，<code>a</code>表示每个数，<code>b</code>表示对应数出现次数，默认为1。每遍历完一批，就对当前
-<code>HashMap</code>进行去重，如果
-<code>b &gt; 1</code>，就排除此数。全部批次进行完筛选之后，剩下的数最后可以直接进行统计，得出不重复的数。</p>
-<h4 id="双间复杂度">双间复杂度</h4>
-<p>时间复杂度：250W * 100轮 + 其他批次。
-这里可以使用多核机器加速运算。</p>
-<p>空间复杂度：int保存数，<code>Key + Value（250W * 4Byte）/ (1024 * 1024)</code>约为
-<code>(Key + 9.5M)</code>内存</p>
-<h3 id="位图法-bitmap">位图法 Bitmap</h3>
-<h4 id="说明-1">说明</h4>
-<p>设计每两个
-<code>bit</code>位，标识一个数的出现情况。如：00表示没有出现，01表示出现一次，10表述出现多次。此题给出的是正整数，所以就是无符号的整型数，占四个字节。</p>
-<p>内存的计算。1B=8b, 4B=32b, 它可以表示的最大整数是<span
-class="math inline">\(2^{32}-1\)</span>, 即需要$2^{32}-1 $约位<span
-class="math inline">\(2^{32}\)</span>个<strong>位</strong>来表示这2.5亿个数，每个状态是两个数，总共就是<span
-class="math inline">\(2^{32}\times 2\)</span>个位。</p>
-<p>一次性申请的位图内存是: $2^{32} $ bit, 即<span
-class="math inline">\(\frac{2^{32}\times2}{(1024\times1024 \times
-8)}=1\)</span> GB。如果加上分治法，甚至都不需要1G内存。</p>
-<p>假设一个数是64, 则对应的bit位是 <span class="math inline">\(64\times
-2=128\)</span>，即127和128共同标识这个数的出现状态（没有出现、出现一次、出现多次）。所以，每当读入了一个数，先找对应的bit位，如果是00就改为01，如果是01就改为10，如果是10就不需要修改。最后扫描整个位图，如果是10，就将下标/2得出出现次数为1的这个数。</p>
-<h2 id="出现次数最多的">出现次数最多的</h2>
+<h4 id="补充">补充</h4>
+<p>当时不知道装CUDA是否需要安装英伟达驱动，所以就安装了。但是安装完之后重启，就不能显示图形界面，原因如问题一。很矛盾的点就在于，如果只安装CUDA不装英伟达驱动，安装CUDA是失败的。综上，不懂如何在AMD的基础上用特斯拉搭建环境。（最后可能怀疑是显卡本身出了问题）</p>
+<p><a
+href="https://developer.nvidia.com/cuda-downloads">CUDA下载地址</a>，这里选择好系统环境，强烈推荐下载<code>runfile</code>，用它进行安装。不推荐安装最新版本。CUDA官网虽声明说CUDA向下兼容，但根据网上反馈和自己的实验来看，CUDA并没有做到。关于主板BIOS设置显卡的问题，实验室这块并不支持。。。。凉凉</p>
+<h4 id="一些实用命令">一些实用命令</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在不能进入图形界面的时候，按住Ctrl+Alt+F1/F2/F3进入终端界面，输入下面的命令卸载英伟达驱动。卸载完重启即可</span></span><br><span class="line">$ <span class="built_in">sudo</span> apt-get remove --purge nvidia-*</span><br><span class="line">$ reboot</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看显卡信息</span></span><br><span class="line">$ lspci | grep -i vga</span><br><span class="line"><span class="comment"># 查看GPU型号</span></span><br><span class="line">$ lspci | grep -i nvidia</span><br><span class="line"><span class="comment"># 查看NVIDIA驱动版本</span></span><br><span class="line">$ <span class="built_in">sudo</span> dpkg --list | grep nvidia-*</span><br></pre></td></tr></table></figure>
+<p>可以参考的一些博客如下：</p>
+<p><a
+href="https://blog.csdn.net/chaihuimin/article/details/71006654?locationNum=2&amp;fps=1">Ubuntu16.04安装NVIDIA显卡驱动和CUDA时的一些坑与解决方案</a></p>
+<p><a href="http://www.cnblogs.com/sp-li/p/7680526.html">ubuntu16.04
+安装NVIDIA和CUDA8.0</a></p>
+<p><a
+href="https://blog.csdn.net/wonengguwozai/article/details/52664597">Ubuntu安装和NVIDIA驱动和安装</a></p>
+<p><a
+href="https://jingyan.baidu.com/article/d8072ac468a91dec95cefdf0.html">主板设置独显/集显</a></p>
+<p>以上，来来回回重装过无数遍系统，耗时5-6天。其中，大部分耗费的时间在于下载和安装，当然还有相当一部分在重装系统的时间上，而且一开始装的是最新的17.10，之后又回退到了16.04，做了几次系统盘重装了无数次系统，并且还只能在课余时间（基本是晚上，和岳岳在实验室折腾）。可以说是相当惨了，后续已经基本熟练到，出了问题直接手敲命令重新来一遍，不需要查资料（熟悉到令人心疼）。</p>
+<h4 id="总结经验教训">总结经验教训</h4>
 <ol type="1">
-<li>找出一篇文章中，出现次数最多的单词</li>
-<li>10亿个正整数找出重复次数最多的100个整数</li>
+<li>踩坑之前，先去请教已经踩过坑的朋友或者去官方论坛查看安装记录或者搜索相关博客。已经两年了，我的习惯还没有改过来QAQ，喜欢自己一遍又一遍踩坑之后再搜索</li>
+<li>提前准备一些工具，比如如果是大型工具，就可能会弄坏（“脏”）系统，备一个系统盘。还有一些能提前获取的压缩包或者安装包。</li>
+<li>踩坑的时候要脚踏实地，不要浮躁冲动。明明不需要重装系统，浪费时间，就不要直接动手格盘重装</li>
 </ol>
-<p>这类问题被叫做Top K问题</p>
-<h3 id="分治法hashmap-1">分治法+HashMap</h3>
-<p>对于文章来说，我们需要规定每批次的字数限制。这里假设100为一批的处理极限，每次处理100个，如果最后一批少于100个就直接当作100来处理。之后每批文字进行切分(<code>split</code>)，比如英文的切割方法可以是空格、标点符号什么的，中文的切割方式可以是空格或者有预先准备好的词库。</p>
-<p>之后就跟上面不重复数字的思路类似了，遍历每批数据，存入
-<code>HashMap&lt;String, Integer&gt;</code>，String表示字符串，Integer对应出现的次数，最大的Integer就是出现次数最多的。</p>
-<p>对数字来说，就是跟不重复数字思路类似，将十亿个数分乘很多份，利用
-<code>HashMap&lt;int, int&gt;</code>来统计，最后找到第二个int最大的那个。每批筛选完之后，就进行排序，选出十亿个数中最大的。</p>
-<h3 id="位图法-bitmap-1">位图法 Bitmap</h3>
-<p>对于文章（字符串）来说，不建议采用</p>
-<p>对于数字来说，此题的背景不需要乘2，将每个数存进位图。出现了一次，bit位=1，没有出现就是0。但是没有办法统计出现次数，所以此题也不适合采用位图法。</p>
-<h2 id="其他">其他</h2>
-<p>在XXX中找出最大、最小、最大的几个、最小的几个。都可以使用分治法+最小堆/最大堆。</p>
-]]></content>
-      <categories>
-        <category>Algorithm</category>
-      </categories>
-      <tags>
-        <tag>HashMap</tag>
-        <tag>位图法</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>Mongodb：副本集配置</title>
-    <url>/hexo_blog/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/</url>
-    <content><![CDATA[<p>Mongodb4.2.x数据库配置的第二部分，副本集架构的配置</p>
-<span id="more"></span>
-<h2 id="准备">准备</h2>
-<p>接
-<a href="/hexo_blog/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/" title="Mongodb从安装到基本操作">Mongodb从安装到基本操作</a>，基本同原文，由于是副本集配置，所以在需要配置的服务器上进行相同的配置</p>
-<p>选择：</p>
+<h3 id="windows10-tesla-amd">Windows10 + Tesla + AMD</h3>
+<p>按照小岳岳的意思，Windows装这种双显卡驱动啥的会比Linux方便很多，所以改战Windows，重装的系统版本是专业版。</p>
+<h4 id="结果-1">结果</h4>
+<p>失败</p>
+<h4 id="遇到的问题">遇到的问题</h4>
+<p>AMD驱动能够正常显示图形并且可以正常安装驱动，但是那块特斯拉显卡不行。系统能识别出来有两块显卡，但是在安装完英伟达驱动之后，并不能设置特斯拉。且两块显卡，AMD显示正常，特斯拉有个黄色的感叹号。所以是否怀疑是否是显卡本身有了问题（毕竟这么久了QAQ）</p>
+<h4 id="补充-1">补充</h4>
+<p>特斯拉显卡本身不是安装一般的英伟达驱动的，而是需要安装<code>TCC</code>驱动程序，<a
+href="http://www.nvidia.cn/object/software-for-tesla-products-cn.html">官网介绍</a>。这里一开始我是不知道的，所以可能是这里出了问题。但是很矛盾的点在于，CUDA在Windows可以直接安装驱动程序。我也试过直接安装CUDA也不行。浪浪的意思是，特斯拉只能在Server版本上运行，我和小岳岳对这一点持怀疑态度。</p>
+<p>关于完整的安装过程就不在这个失败部分介绍了，下一节完整的说明Windows下安装过程及注意事项。此部分耗时2-3天。</p>
+<h3 id="windows10-titan-x">Windows10 + Titan X</h3>
+<p>由于以上种种原因，并且已经快折腾不下去了，所以跟教授直接申请换泰坦。嗯，所以这一波一天半大概就成功了，只失败了一次。泰坦是支持图形显示的，为了减少问题所以也将AMD换下了（第一次没卸，出了一点问题，可能跟AMD有关）。</p>
+<h4 id="软件版本">软件版本</h4>
+<p>Windows10专业版+VS2013专业版+CUDA7.5</p>
+<p>这里需要注意的是，在前文中提到CUDA不向下兼容，所以推荐不安装最新的，保险起见就选了较旧的版本。关于VS版本选择问题，这里CUDA7.5网上最多的搭配是VS2013，但是这里需要说明VS2013必须要是专业版，否则不支持CUDA。在VS2015之后的版本就没有这个问题。还有就是顺序问题，<strong>一定要先安装VS再安装CUDA</strong>，不需要安装英伟达驱动。</p>
+<h4 id="总体流程">总体流程</h4>
 <ol type="1">
-<li>单机上，配置不同port的mongo服务</li>
-<li>不同机器上，配置mongo服务</li>
-<li>要求同网段，所以配置的ip中不能写127.0.0.1</li>
-<li>可以 <code>vim /etc/hosts</code>，方便管理</li>
+<li><p>安装Windows系统，激活系统，最好不进行安全更新。卸载一些不必要的软件防止占用空间大小（比如说我只有一块128G的固态，惨！）</p></li>
+<li><p>安装VS2013专业版</p></li>
+<li><p>安装CUDA7.5，建议local版。如果这里安装失败可以进入<strong>服务</strong>，找到<strong>Windows
+Installer</strong>，手动启动它即可。还有一些其他问题，在文末贴了链接。</p></li>
+<li><p>写CUDA的环境配置。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># cuda7.5安装完成之后在系统环境变量中自动配置了两个系统变量</span></span><br><span class="line">CUDA_PATH：C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5</span><br><span class="line">CUDA_PATH_V7_5：C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5</span><br><span class="line"><span class="comment"># 但是为了之后的vs2013的配置做准备我们需要在配置五个系统变量，请根据具体的安装路径进行设置CUDA_SDK_PATH, 这里我用的是默认设置</span></span><br><span class="line">CUDA_BIN_PATH：%CUDA_PATH%\bin</span><br><span class="line">CUDA_LIB_PATH：%CUDA_PATH%\lib\Win32</span><br><span class="line">CUDA_SDK_BIN：%CUDA_SDK_PATH%\bin\Win64</span><br><span class="line">CUDA_SDK_LIB：%CUDA_SDK_PATH%\common\lib\x64</span><br><span class="line">CUDA_SDK_PATH：C:\ProgramData\NVIDIA Corporation\CUDA Samples\v7.5</span><br><span class="line"><span class="comment"># 在系统环境变量path后添加如下内容</span></span><br><span class="line">%CUDA_LIB_PATH%</span><br><span class="line">%CUDA_BIN_PATH%</span><br><span class="line">%CUDA_SDK_LIB_PATH%</span><br><span class="line">%CUDA_SDK_BIN_PATH%</span><br></pre></td></tr></table></figure></li>
+<li><p>进行测试</p></li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载压缩包</span></span><br><span class="line">wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-4.2.15.tgz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxvf mongodb-linux-x86_64*.tgz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重命名</span></span><br><span class="line"><span class="built_in">mv</span> mongodb-linux-x86_64* mongodb</span><br><span class="line"><span class="comment"># 移动到你想放置的目录下，这里我放在/opt目录下</span></span><br><span class="line"><span class="built_in">mv</span> mongodb /opt/</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行mongodb主目录</span></span><br><span class="line"><span class="built_in">cd</span> /opt/</span><br><span class="line"><span class="comment"># 建立存放数据文件和日志文件的目录</span></span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/logs</span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/db</span><br><span class="line"><span class="comment"># 创建配置文件，并写入如下配置</span></span><br><span class="line">vim bin/mongodb.conf</span><br></pre></td></tr></table></figure>
-<p>更新部分，配置文件用yaml格式</p>
-<figure class="highlight yaml"><table><tr><td class="code"><pre><span class="line"><span class="attr">systemLog:</span></span><br><span class="line">  <span class="attr">destination:</span> <span class="string">file</span></span><br><span class="line">  <span class="attr">path:</span> <span class="string">&quot;/opt/mongodb/data/test/logs/mongodb.log&quot;</span></span><br><span class="line">  <span class="attr">logAppend:</span> <span class="literal">true</span></span><br><span class="line"><span class="attr">storage:</span></span><br><span class="line">  <span class="attr">dbPath:</span> <span class="string">&quot;/opt/mongodb/data/test/db&quot;</span></span><br><span class="line">  <span class="attr">journal:</span></span><br><span class="line">    <span class="attr">enabled:</span> <span class="literal">true</span> </span><br><span class="line"><span class="attr">net:</span></span><br><span class="line">  <span class="comment"># 允许所有IP访问，也可以用逗号隔开，控制IP访问</span></span><br><span class="line">  <span class="attr">bindIp:</span> <span class="number">0.0</span><span class="number">.0</span><span class="number">.0</span> </span><br><span class="line">  <span class="comment"># 设置端口号（默认的端口号是27017，可以根据个人需求进行更改）</span></span><br><span class="line">  <span class="attr">port:</span> <span class="number">27017</span></span><br><span class="line"><span class="attr">processManagement:</span></span><br><span class="line">  <span class="comment"># 设置为以守护进程的方式运行，即在后台运行</span></span><br><span class="line">  <span class="attr">fork:</span> <span class="literal">true</span></span><br><span class="line"><span class="attr">replication:</span></span><br><span class="line">  <span class="comment"># 副本集名字</span></span><br><span class="line">  <span class="attr">replSetName:</span> <span class="string">&quot;rs0&quot;</span> </span><br></pre></td></tr></table></figure>
-<h3 id="启动">启动</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在所有机器上启动mongo，此时无账号密码</span></span><br><span class="line"><span class="comment"># 以配置文件的方式启动</span></span><br><span class="line">bin/mongod --config mongodb.conf</span><br></pre></td></tr></table></figure>
-<h3 id="配置副本集">配置副本集</h3>
-<p>在任意一台服务器A上，进行配置副本集</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 将副本集的初始化配置写入js</span></span><br><span class="line">vim init_replica.js</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 其中rs0一定要跟上面的replSetName一样</span></span><br><span class="line">config = &#123; _id:<span class="string">&quot;rs0&quot;</span>, members:[&#123;_id:0,host:<span class="string">&quot;ip0:port0&quot;</span>&#125;,&#123;_id:1,host:<span class="string">&quot;ip1:port1&quot;</span>&#125;]&#125;</span><br><span class="line">rs.initiate(config) </span><br><span class="line">rs.status()</span><br></pre></td></tr></table></figure>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/mongod --port 27017 &lt; init_replica.js</span><br></pre></td></tr></table></figure>
-<h3 id="验证">验证</h3>
-<p>在A服务器上进行操作，添加账号密码</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb的交互环境</span></span><br><span class="line">bin/mongod --port 27017</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行admin数据库，创建管理员用户root，密码为password，权限是超级用户（最高）</span></span><br><span class="line">&gt;use admin</span><br><span class="line">&gt;db.createUser(&#123; user: <span class="string">&quot;admin&quot;</span>, <span class="built_in">pwd</span>: <span class="string">&quot;password&quot;</span>, roles: [&#123; role: <span class="string">&quot;userAdminAnyDatabase&quot;</span>, db: <span class="string">&quot;admin&quot;</span> &#125;] &#125;)</span><br><span class="line">&gt;db.createUser(&#123;user:<span class="string">&quot;root&quot;</span>,<span class="built_in">pwd</span>:<span class="string">&quot;password&quot;</span>,roles:[<span class="string">&quot;root&quot;</span>]&#125;)</span><br><span class="line"><span class="comment"># 验证是否创建成功，返回1表示成功</span></span><br><span class="line">&gt;db.auth(&#123;<span class="string">&quot;root&quot;</span>, <span class="string">&quot;password&quot;</span>&#125;)</span><br><span class="line"><span class="comment"># 退出交互环境</span></span><br><span class="line">&gt;<span class="built_in">exit</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找到mongo的服务进程pid</span></span><br><span class="line">ps ux | grep mongod</span><br><span class="line"><span class="comment"># 杀死进程，填写对应的pid</span></span><br><span class="line"><span class="built_in">kill</span> PID</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以密码权限验证启动服务</span></span><br><span class="line">bin/mongod --config mongodb.conf --auth</span><br></pre></td></tr></table></figure>
-<h3 id="openssl证书文件">OpenSSL证书文件</h3>
-<h3 id="读写分离">读写分离</h3>
-<h3 id="主节点选举配置">主节点选举配置</h3>
+<h4 id="参考文章">参考文章</h4>
+<p><a
+href="https://blog.csdn.net/u011821462/article/details/50145221">win10+vs2013+cuda7.5环境搭建</a></p>
+<p><a
+href="https://blog.csdn.net/u012033124/article/details/52169823">VS2013下CUDA
+7.5安装</a></p>
+<p>CUDA安装的一些错误可以参考<a
+href="https://blog.csdn.net/xuxiatian/article/details/50577960">这篇文章</a>，但是在某些情况下，可能系统环境已经被污染了，最好的方法还是重装系统。</p>
+<p>耗时1-2天，总归是很愉快的结束了整个环境搭建。在某些情况下，有些环境的搭建不一定需要（最好不要）用最新的版本，可能会有一些bug，而且之前踩坑的人比较少可借鉴的东西较少。总体来说，抛开之前很奇怪的硬件配置之外，安装过程还是并不难的。接下来，便开始了我的CUDA编程之旅。</p>
+<p>人生路漫漫，谨以此文章纪念上大学来折腾环境最坑爹的一次记录。</p>
 ]]></content>
       <categories>
-        <category>Database</category>
+        <category>Environment</category>
       </categories>
       <tags>
-        <tag>mongodb</tag>
-        <tag>数据库</tag>
+        <tag>GPU</tag>
+        <tag>CUDA</tag>
       </tags>
   </entry>
   <entry>
-    <title>Mongodb从安装到基本操作</title>
-    <url>/hexo_blog/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/</url>
-    <content><![CDATA[<p>这篇文章主要从零开始介绍如何在Centos7下安装Mongodb,
-用Python3连接Mongodb, 用可视化工具Robo 3T连接Mongodb,
-之后介绍了一些基本的mongo操作方法, 其中含有一些进阶操作</p>
+    <title>OpenWrt路由器折腾记录</title>
+    <url>/hexo_blog/2023/01/10/IOT/OpenWrt%E8%B7%AF%E7%94%B1%E5%99%A8%E8%AE%B0%E5%BD%95/</url>
+    <content><![CDATA[<p>在路由器上安装和配置OpenWrt。它包括安装软件包和设置防火墙、接口和各种网络连接的分步说明。</p>
 <span id="more"></span>
-<h2 id="安装mongodbcentos7">安装Mongodb(Centos7)</h2>
-<ol type="1">
-<li>官网上找到需要下载的版本压缩包进行下载，<a
-href="https://fastdl.mongodb.org/linux/">官网链接</a>。或者在命令行下使用<code>wget</code>命令进行下载</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 这里下载的是3.4.10版本</span></span><br><span class="line">wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel70-3.4.10.tgz</span><br></pre></td></tr></table></figure>
-<ol start="2" type="1">
-<li>进行解压缩</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxvf mongodb-linux-x86_64*.tgz</span><br><span class="line"><span class="comment"># 重命名</span></span><br><span class="line"><span class="built_in">mv</span> mongodb-linux-x86_64* mongodb</span><br><span class="line"><span class="comment"># 移动到你想放置的目录下，这里我放在/opt目录下</span></span><br><span class="line"><span class="built_in">mv</span> mongodb /opt/</span><br></pre></td></tr></table></figure>
-<ol start="3" type="1">
-<li>进行相应的配置，mongodb默认没有任何配置</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb主目录</span></span><br><span class="line"><span class="built_in">cd</span> /opt/</span><br><span class="line"><span class="comment"># 建立存放数据文件和日志文件的目录</span></span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/logs</span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/db</span><br><span class="line"><span class="comment"># 创建配置文件，并写入如下配置</span></span><br><span class="line">vim bin/mongodb.conf</span><br><span class="line">  `</span><br><span class="line">    <span class="comment"># 设置数据文件的存放目录</span></span><br><span class="line">    dbpath = /opt/mongodb/data/test/db</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置日志文件的存放目录及其日志文件名</span></span><br><span class="line">    logpath = /opt/mongodb/data/test/logs/mongodb.log</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置端口号（默认的端口号是27017，可以根据个人需求进行更改）</span></span><br><span class="line">    port = 27017</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置为以守护进程的方式运行，即在后台运行</span></span><br><span class="line">    fork = <span class="literal">true</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 是否不允许表扫描</span></span><br><span class="line">    nohttpinterface = <span class="literal">true</span></span><br><span class="line">  `</span><br><span class="line"><span class="comment"># 保存退出</span></span><br></pre></td></tr></table></figure>
-<ol start="4" type="1">
-<li>启动mongodb</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 以配置文件的方式启动</span></span><br><span class="line">./bin/mongod --config mongodb.conf</span><br></pre></td></tr></table></figure>
-<p>报错一：
-<code>ERROR: child process failed, exited with error number 1</code>
-检查mongodb.conf的文件路径是否配置错误</p>
-<p>报错二：<code>ERROR: child process failed, exited with error number 100</code>
-很可能是没有正常关闭导致的，那么可以删除
-<code>mongod.lock</code>文件，这里对应我的配置路径在<code>data/</code>里面</p>
-<ol start="5" type="1">
-<li>其他</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 链接命令，方便调用mongo命令</span></span><br><span class="line"><span class="built_in">ln</span> -s /opt/mongodb/bin/mongo /usr/bin</span><br><span class="line"><span class="comment"># 查看mongodb进程</span></span><br><span class="line">ps -aux |grep mongodb</span><br><span class="line"><span class="comment"># 检查端口运行情况</span></span><br><span class="line">netstat -lanp | grep 27017</span><br><span class="line"><span class="comment"># 终止mongodb服务，PID从ps命令获取</span></span><br><span class="line"><span class="built_in">kill</span> -15 PID</span><br><span class="line"><span class="comment"># 添加自启动命令</span></span><br><span class="line">vim /etc/rc.local</span><br><span class="line"><span class="comment"># 末尾追加一行</span></span><br><span class="line">`/opt/mongodb/bin/mongod --config mongodb.conf`</span><br><span class="line"><span class="comment"># 保存退出</span></span><br></pre></td></tr></table></figure>
-<ol start="6" type="1">
-<li>设置密码权限，默认无密码</li>
-</ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb的交互环境</span></span><br><span class="line">./mongo</span><br><span class="line"><span class="comment"># 如果进入失败，请检查是否添加了软链接和是否启动了mongodb服务</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行admin数据库，创建管理员用户root，密码为password，权限是超级用户（最高）</span></span><br><span class="line">&gt;use admin</span><br><span class="line">&gt;db.createUser(&#123;user:<span class="string">&quot;root&quot;</span>,<span class="built_in">pwd</span>:<span class="string">&quot;password&quot;</span>,roles:[<span class="string">&quot;root&quot;</span>]&#125;)</span><br><span class="line"><span class="comment"># 验证是否创建成功，返回1表示成功</span></span><br><span class="line">&gt;db.auth(&#123;<span class="string">&quot;root&quot;</span>, <span class="string">&quot;password&quot;</span>&#125;)</span><br><span class="line"><span class="comment"># 退出交互环境</span></span><br><span class="line">&gt;<span class="built_in">exit</span></span><br><span class="line"><span class="comment"># 重启mongodb服务</span></span><br><span class="line"><span class="comment"># 杀死mongodb进程，参照上面的方法</span></span><br><span class="line"><span class="comment"># 以密码权限验证启动服务</span></span><br><span class="line">opt/mongodb/mongod --config mongodb.conf --auth</span><br></pre></td></tr></table></figure>
-<h2 id="使用python连接mongodb">使用Python连接Mongodb</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载第三方包</span></span><br><span class="line">pip3 install pymongo</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># /usr/bin/python3</span></span><br><span class="line"><span class="keyword">from</span> pymongo <span class="keyword">import</span> MongoClient</span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line"><span class="comment"># host主机名，27017连接端口</span></span><br><span class="line">client = MongoClient(host, <span class="number">27017</span>)</span><br><span class="line">db_auth = client.admin</span><br><span class="line"><span class="comment"># 登陆的用户名(username)和密码(password)</span></span><br><span class="line">db_auth.authenticate(username, password)</span><br><span class="line"><span class="comment"># 连接指定数据库，数据库名为db_name</span></span><br><span class="line">db_name = client[<span class="string">&quot;db_name&quot;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line"><span class="comment"># 用户名(username)、密码(password)、主机名(host)、端口(port)。注：这里的password不能出现@符号，如果用@符号就需要使用方法一</span></span><br><span class="line">client = MongoClient(<span class="string">&quot;mongodb://username:password@host:port&quot;</span>)</span><br><span class="line"><span class="comment"># 连接指定数据库，数据库名为db_name</span></span><br><span class="line">db_name = client[<span class="string">&quot;db_name&quot;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 操作数据库，db_name[&quot;db_set&quot;]=db_name.db_set</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">查找数据</span></span><br><span class="line"><span class="string">在db_name数据库中的db_set集合里面找到name是a_name的json</span></span><br><span class="line"><span class="string">find是返回一个指针（可以使用列表的方式来读取），find_one返回一个dict</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line">db_name[<span class="string">&quot;db_set&quot;</span>].find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;a_name&quot;</span>&#125;)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 插入数据</span></span><br><span class="line"><span class="comment"># 插入一个json到set1集合中，a_dict是一个字典（如果set1不存在则会被自动创建）</span></span><br><span class="line">db_name.set1.insert_one(a_dict)</span><br><span class="line"><span class="comment"># 插入多个json到set1集合中，a_dict_lst是一个列表，里面的元素是由字典组成的</span></span><br><span class="line">db_name.set1.insert_many(a_dict_lst)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 更新数据</span></span><br><span class="line"><span class="comment"># 查找到一个数据之后，对里面的某个数据进行更改，再使用save方法保存</span></span><br><span class="line">data_one = db_name[<span class="string">&quot;db_set&quot;</span>].find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;a_name&quot;</span>&#125;)</span><br><span class="line">data_one[<span class="string">&#x27;age&#x27;</span>] += <span class="number">3</span></span><br><span class="line">db_name.db_set.save(data_one)</span><br><span class="line"><span class="comment"># 更新多条记录，找到db_set集合中满足age=20，sex=0的数据，将name改为user1</span></span><br><span class="line">db_name.db_set.update(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;, &#123;<span class="string">&quot;$set&quot;</span>:&#123;<span class="string">&quot;age&quot;</span>: <span class="number">20</span>, <span class="string">&quot;sex&quot;</span>: <span class="number">0</span>&#125;&#125;)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除数据</span></span><br><span class="line"><span class="built_in">id</span> = db_name.db_set.find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;)[<span class="string">&quot;_id&quot;</span>]</span><br><span class="line"><span class="comment"># 根据 id 删除一条记录</span></span><br><span class="line">db_name.db_set.remove(<span class="built_in">id</span>)</span><br><span class="line"><span class="comment"># 删除集合里的所有记录</span></span><br><span class="line">db_name.db_set.remove()</span><br><span class="line"><span class="comment"># 删除name=user1的记录</span></span><br><span class="line">db_name.db_set.remove(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;)</span><br></pre></td></tr></table></figure>
-<h2 id="使用robo-3t连接mongodb">使用Robo 3t连接Mongodb</h2>
-<p>强烈推荐Robo 3t，<a href="https://robomongo.org/">官网地址</a></p>
-<ol type="1">
-<li><p>到官网找到对应的系统版本进行下载</p></li>
-<li><p>安装</p>
+<h2 id="基本界面">基本界面</h2>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673353557758.png" /></p>
+<p>准备：</p>
 <ul>
-<li><p>Windows的安装十分简单就不赘述了</p></li>
-<li><p>Ubuntu的安装 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxzf robomongo*.tar.gz</span><br><span class="line"><span class="comment"># 修改文件名（可选）</span></span><br><span class="line"><span class="built_in">mv</span> robomongo* robo3t</span><br><span class="line"><span class="comment"># 移动到/opt目录（可选）</span></span><br><span class="line"><span class="built_in">mv</span> robo3t /opt</span><br><span class="line"><span class="comment"># 建立软链接</span></span><br><span class="line"><span class="built_in">ln</span> -s /opt/robo3t/robomongo /usr/bin</span><br><span class="line"><span class="comment"># 启动</span></span><br><span class="line">robomongo</span><br></pre></td></tr></table></figure> <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 如果报如下错误</span></span><br><span class="line">`</span><br><span class="line">This application failed to start because it could not find or load the Qt platform plugin <span class="string">&quot;xcb&quot;</span> <span class="keyword">in</span> <span class="string">&quot;&quot;</span>.</span><br><span class="line"></span><br><span class="line">Available platform plugins are: xcb.</span><br><span class="line"></span><br><span class="line">Reinstalling the application may fix this problem.</span><br><span class="line">Aborted (core dumped)</span><br><span class="line">`</span><br><span class="line"><span class="comment"># 解决办法</span></span><br><span class="line"><span class="built_in">mkdir</span> ~/robo-backup</span><br><span class="line"><span class="built_in">mv</span> /opt/robo3t/lib/libstdc++* ~/robo-backup/</span><br><span class="line"><span class="comment"># 之后便可以重新启动mongo试试看了</span></span><br></pre></td></tr></table></figure></p></li>
+<li>固件</li>
+<li>待恢复的配置</li>
+<li>需要安装的软件+运行的命令</li>
+</ul>
+<p>配置出问题，直接重刷。同时方便起见，要及时备份当前的配置。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673353799884.png" /></p>
+<h2 id="软件包的安装">软件包的安装</h2>
+<p>能在线安装的可以直接在线安装。直接从"系统-&gt;软件包"里面找到需要安装的软件。</p>
+<p>但有时候在线安装会由于网络，软件源、软件版本等问题，无法自动安装。因此需要我们手动找到ipk文件进行安装。使用下面的命令进行安装。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">opkg install XXX</span><br></pre></td></tr></table></figure>
+<p>官方软件下载网页：<a
+href="https://downloads.openwrt.org/releases/">https://downloads.openwrt.org/releases/</a></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 系统版本可以直接从管理界面的概况中得知，这里是17.01.4</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 内核版本</span></span><br><span class="line">&gt;&gt;&gt; <span class="built_in">uname</span> -a </span><br><span class="line">Linux dw33d 4.4.194 <span class="comment">#0 Tue Oct 17 17:46:20 2017 mips GNU/Linux</span></span><br><span class="line"><span class="comment"># 查看支持的架构</span></span><br><span class="line">&gt;&gt;&gt; opkg print-architecture</span><br><span class="line"><span class="built_in">arch</span> all 1</span><br><span class="line"><span class="built_in">arch</span> noarch 1</span><br><span class="line"><span class="built_in">arch</span> mips_24kc 10</span><br></pre></td></tr></table></figure>
+<p>对应我这里的第三方包就是：<a
+href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/packages/">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/packages/</a></p>
+<p>基础的库：<a
+href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/base">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/base</a></p>
+<p>luci：<a
+href="https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/luci/">https://downloads.openwrt.org/releases/17.01.4/packages/mips_24kc/luci/</a></p>
+<h3 id="tmux">tmux</h3>
+<p>以tmux安装为例，该软件无法直接安装。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 1. 从package里面下载tmux的ipkg文件</span></span><br><span class="line"><span class="comment"># 2. 执行安装命令</span></span><br><span class="line">&gt;&gt;&gt;opkg install tmux_2.3-1_mips_24kc.ipk</span><br><span class="line"><span class="comment"># 3. 报错信息，一些依赖文件没有。所以从base里面找到对应缺失的ipk文件。并进行安装</span></span><br><span class="line">&gt;&gt;&gt;opkg install terminfo_6.0-1_mips_24kc.ipk libncurses_6.0-1_mips_24kc.ipk</span><br><span class="line"><span class="comment"># 4. 此时，再安装tmux。安装成功</span></span><br><span class="line">&gt;&gt;&gt;opkg install tmux_2.3-1_mips_24kc.ipk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 但安装libevent2这个库时，提示当前已安装，并且版本过高，并不会自动降级。所以需要手动卸载原有的软件</span></span><br><span class="line">&gt;&gt;&gt;opkg remove libevent2 --force-depends</span><br><span class="line"><span class="comment"># 卸载失败，提示已有软件依赖该库。所以我们需要强制进行卸载，将依赖软件也进行卸载。加上--force 命令</span></span><br><span class="line"><span class="comment"># 之后再安装低版本的libncurses</span></span><br><span class="line">&gt;&gt;&gt;opkg install libevent2_2.0.22-1_mips_24kc.ipk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最后再运行tmux，成功</span></span><br></pre></td></tr></table></figure>
+<p>其他软件应该同理，需要一步一步找到依赖文件，并且耐心地解决各种问题。dw33d内置了一个16g的sd卡，所以我把一些重装必备的ipk文件放到里面，便于快速安装。并且刷写固件并不会影响该sd卡的内容，可以放心存储。</p>
+<h3 id="tailscale">Tailscale</h3>
+<p>异地组网工具，从官网下载找到mips版本，放到sd卡中，避免刷新固件有影响。相当于单文件就可以运行。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压文件</span></span><br><span class="line">&gt;&gt;&gt; tar zxvf tailscale_1.34.1_mips.tgz</span><br><span class="line"><span class="comment"># 到对应的目录中，里面会有一些文件。主要是两个可执行文件tailscale和tailscaled。</span></span><br><span class="line"><span class="comment"># 第一次运行需要启动tailscaled服务</span></span><br><span class="line">&gt;&gt;&gt;./tailscaled --state=tailscaled.state</span><br><span class="line"><span class="comment"># 此时需要新开一个窗口，登录tailscale</span></span><br><span class="line">&gt;&gt;&gt;./tailscale login </span><br><span class="line"><span class="comment"># 可以直接把网址复制出来，用其他设备的浏览器进行登录</span></span><br></pre></td></tr></table></figure>
+<p>登录完毕之后，组网就完成了。此时，可以关闭该窗口。但为了后台启动，所以可以使用tmux，或者nohup命令。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">nohup</span> ./tailscaled --state=tailscaled.state &gt; t.log 2&gt;&amp;1 &amp;</span><br></pre></td></tr></table></figure>
+<p>同时为了保证开机自启动，所以我们可以在web管理界面中"系统-&gt;启动项"中，最后的"本地启动脚本"中插入(<code>exit 0</code>之前行)</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 先切换到对应目录，再执行</span></span><br><span class="line"><span class="built_in">cd</span> /mnt/sda1/tailscale_1.34.1_mips &amp;&amp; <span class="built_in">nohup</span> ./tailscaled --state=tailscaled.state &gt; t.log 2&gt;&amp;1 &amp;</span><br></pre></td></tr></table></figure>
+<p>这样，就能保证异地管理该路由器的web界面，并且可以直接ssh登录</p>
+<h2 id="防火墙">防火墙</h2>
+<p>"网络-&gt;防火墙"中的默认配置，wan口是拒绝入站数据的。这样就会导致如果该路由器跟其他设备同属于一个局域网，ssh和web界面均不能通过ip登录。具体的例子是：</p>
+<ul>
+<li>该路由器连接了一个上级路由器A，获取到的IP是192.168.0.X。分配给下面的设备的IP是192.168.1.X</li>
+<li>另一个设备接入路由器A时，获取到的IP是192.168.0.Y。此时能ping通192.168.0.X，但是无法进行ssh，也无法通过192.168.0.X访问管理界面</li>
+<li>另一个设备接入该路由器时，获取到的IP是192.168.1.Y。此时能够通过ssh和web访问192.168.1.X。</li>
+</ul>
+<p>需求：在同局域网下的其他设备也能访问该路由器。</p>
+<p>做法：</p>
+<ul>
+<li>一个简单的做法就是如下图所示，把"防火墙"中的"常规设置"里面的wan口，入站数据从<strong>拒绝</strong>改成<strong>接受</strong>。（默认是拒绝入站的）</li>
+</ul>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1680509223.png" /></p>
+<ul>
+<li>另一个做法是配置"防火墙-&gt;端口转发"，把需要配置的端口转发出来。如下图所示，web界面的端口是80。所以从wan口出来的访问80端口的流量，直接转发到192.168.4.1（该路由器分配给下面的IP，不是获取的IP）</li>
+</ul>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673356610477.png" /></p>
+<p>关于流量规则这里，则是在系统内部决定是否开放某个端口。可以参考默认的一些配置进行开放，一般应该是不需要调整的。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673356804612.png" /></p>
+<h2 id="接口">接口</h2>
+<p>"网络-&gt;接口"</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357070459.png" /></p>
+<p>我主要会有以下四种上网方式（不会专业术语，简单描述现象）：</p>
+<ul>
+<li><p>桥接光猫</p>
+<ul>
+<li>将路由器WAN口接入光猫LAN口，代替光猫进行拨号，需要知道宽带的账号密码。</li>
+<li>在WAN口中，协议选择<strong>PPPoE</strong>，输入账号密码</li>
+<li>此时该路由器就是主路由器，一般同时有ipv4和ipv6两个地址，假设ipv4的ip为192.168.1.1</li>
+<li>接入该路由器的设备，由该路由器进行分配ip，ip与路由器同网段，如192.168.1.2</li>
 </ul></li>
-<li><p>使用</p>
-<ol type="1">
-<li>首先创建新的连接，如下图所示。在<code>Connection中，``Name</code>为自定义连接名，<code>Address</code>填入对应的<code>host</code>和<code>port</code>；
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/mongodb/20171124171027733.png" /></li>
-<li>在<code>Authentication</code>中，勾选<code>Perform authentication</code>，填上对应的<code>User Name</code>和<code>Password</code>。填写完成之后可以点击<code>Test</code>验证是否能够连接成功，如果无误的话，可以点击<code>Save</code>保存退出。</li>
-</ol>
+<li><p>桥接上一级路由器</p>
+<ul>
+<li>将路由器WAN口接入上一级路由器LAN口，无需知道账号密码</li>
+<li>在WAN口中，协议选择<strong>DHCP客户端</strong></li>
+<li>在物理设置中，选择桥接接口，默认<strong>以太网交换机:
+eth0</strong>开启。勾选两个无线网络。如下图所示</li>
+<li><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673524264094.png" /></li>
+<li>此时，IP同上一级路由器同网段，且如果有IPv6，则连接该路由器的设备也有IPv6</li>
+</ul></li>
+<li><p>做NAT</p>
+<ul>
+<li>将路由器WAN接入上一级路由器的LAN口，相当于把该路由器做一个新的入口。</li>
+<li>在WAN中，协议选择<strong>DHCP客户端</strong>或<strong>静态地址</strong>（需要额外配置）</li>
+<li>此时，接入该路由器的设备会独立上一级路由器的设备，即网段不同。</li>
+<li>注：如果此时还需要ipv6，可以参考这篇文件进行配置。<a
+href="https://www.lategege.com/?p=676">https://www.lategege.com/?p=676</a>
+<ul>
+<li>简单来说，就是需要选择静态IP，并在<strong>WAN口</strong>和<strong>LAN</strong>中的"DHCP服务器"中的IPv6设置中，路由器广告服务，DHCPv6服务，NDP-代理三个均选择中继模式</li>
+<li>并且在终端中 <code>vim /etc/config/dhcp</code>，找到
+<code>config dhcp 'wan'</code>对应的选项，加入
+<code>option master '1'</code>这一行</li>
+</ul></li>
+</ul></li>
+<li><p>WiFi放大器</p>
+<ul>
+<li><p>无需网线</p></li>
+<li><p>"网络-&gt;无线"中的搜索，找到需要放大WiFi，加入网络。看情况选择放大2.4G还是5G的WiFi</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/mongodb/20171124171111807.png" /></p>
-<ol start="3" type="1">
-<li>连接Mongodb。点击<code>Connect</code>，之后就可以可视化数据库中的数据。也可以使用鼠标简单的进行创建/删除数据库，新建/修改信息，也可以使用命令查找数据。</li>
-</ol></li>
-</ol>
-<h2 id="curd操作主要是查找数据">CURD操作（主要是查找数据）</h2>
-<p><strong>CRUD</strong></p>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357519480.png" /></p></li>
+<li><p>WPA密钥就是WiFi密码，新的网络名称可以默认也可以另起一个以便区分。</p></li>
+<li><p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/openwrt/1673357576678.png" /></p></li>
+<li><p>此时接口中会多一个接口</p></li>
+<li><p>其他同桥接</p></li>
+</ul></li>
+</ul>
+]]></content>
+      <categories>
+        <category>Linux</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>路由器</tag>
+        <tag>OpenWrt</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>安卓手机利用Termux安装Linux</title>
+    <url>/hexo_blog/2021/07/21/IOT/%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA%E5%88%A9%E7%94%A8Termux%E5%AE%89%E8%A3%85Linux/</url>
+    <content><![CDATA[<p>本文介绍如何在安卓手机上安装和运行Ubuntu系统，并通过ssh连接进行操作，同时给出了一些其他有用的命令和配置说明。</p>
+<span id="more"></span>
+<h3 id="安装">安装</h3>
+<p>不要从Google Play上安装，正确的安装包地址<a
+href="https://f-droid.org/packages/com.termux/">f-droid</a></p>
+<p>下载完成后，正常安装，但需要额外的给存储空间权限。</p>
+<p>虽然说可以免root安装，但没有root权限，有些操作不能很好的完成，如
+<code>top</code></p>
+<h3 id="连接">连接</h3>
+<p>在手机上配置太麻烦，所以可以使用同一局域网下的电脑进行配置</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 手机上</span></span><br><span class="line"><span class="comment"># 安装ssh服务</span></span><br><span class="line">pkg install openssh -y</span><br><span class="line"><span class="comment"># 开启ssh服务器</span></span><br><span class="line">sshd</span><br><span class="line"><span class="comment"># 配置密码</span></span><br><span class="line">passwd</span><br><span class="line"><span class="comment"># 输入想要配置的密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 电脑终端, ip填写对应的手机ip，8022是termux默认的ssh端口</span></span><br><span class="line">ssh root@[ip] -p8022</span><br><span class="line"><span class="comment"># 输入刚刚的密码</span></span><br></pre></td></tr></table></figure>
+<p>至此，就能在电脑端进行配置</p>
+<h3 id="配置">配置</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 安装第三方库</span></span><br><span class="line">pkg install proot git python wget -y</span><br><span class="line"></span><br><span class="line"><span class="comment"># 一键安装项目</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/sqlsec/termux-install-linux.git</span><br><span class="line"><span class="built_in">cd</span> termux-install-linux</span><br><span class="line">python termux-linux-install.py</span><br><span class="line"><span class="comment"># 选择Ubuntu</span></span><br><span class="line"><span class="built_in">cd</span> ~/Termux-Linux/Ubuntu</span><br><span class="line">./start-ubuntu.sh</span><br><span class="line"></span><br><span class="line"><span class="comment"># 或者，ubuntu版本，最新支持20.04</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/MFDGaming/ubuntu-in-termux</span><br><span class="line"><span class="built_in">cd</span> ubuntu-in-termux</span><br><span class="line"><span class="built_in">chmod</span> +x ubuntu.sh</span><br><span class="line">./ubuntu.sh -y</span><br><span class="line">./startubuntu.sh</span><br></pre></td></tr></table></figure>
+<p>这样安装的Linux是与Termux共享文件目录的，所以可以直接将文件拷贝在
+<code>ubuntu-fs/root</code>文件夹中，启动
+<code>./startubuntu.sh</code>后，就能直接看到文件。</p>
+<p>注：若进入系统后，无法更新软件源，可能是dns存在问题，可以在termux中更改dns解析（因为系统中不好编辑）。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim ubuntu-fs/etc/resolv.conf</span><br><span class="line"></span><br><span class="line">`</span><br><span class="line">nameserver 114.114.114.114</span><br><span class="line">nameserver 8.8.8.8</span><br><span class="line">`</span><br><span class="line"><span class="comment"># 若依旧不行，可尝试其他dns，如1.1.1.1等</span></span><br></pre></td></tr></table></figure>
+<h3 id="其他">其他</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">pkg install vim tmux -y</span><br><span class="line"></span><br><span class="line"><span class="comment"># frp项目地址，手机请注意下载arm版本，一般为arm64，不能直接在Linux系统中配置，必须得在Termux中操作</span></span><br><span class="line">[https://github.com/fatedier/frp/releases](https://github.com/fatedier/frp/releases)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机温度，需要除以1000</span></span><br><span class="line"><span class="built_in">cat</span> /sys/class/thermal/thermal_zone0/temp</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机电池容量，需root</span></span><br><span class="line"><span class="built_in">cat</span> /sys/class/power_supply/battery/capacity</span><br><span class="line"></span><br><span class="line"><span class="comment"># 配置termux的启动操作</span></span><br><span class="line">vim ~/.bashrc</span><br><span class="line"></span><br><span class="line">`</span><br><span class="line">sshd <span class="comment"># 开启ssh服务，便于连接</span></span><br><span class="line"><span class="comment"># 配置frp相关操作，便于直接远程连接</span></span><br><span class="line"><span class="comment"># 配置进入Linux系统操作</span></span><br><span class="line">`</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Android</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>Android</tag>
+        <tag>Termux</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>对各种电子产品的拓展玩法与互联</title>
+    <url>/hexo_blog/2019/09/14/IOT/%E5%AF%B9%E5%90%84%E7%A7%8D%E7%94%B5%E5%AD%90%E8%AE%BE%E5%A4%87%E7%9A%84%E6%8B%93%E5%B1%95%E7%8E%A9%E6%B3%95%E4%B8%8E%E4%BA%92%E8%81%94/</url>
+    <content><![CDATA[<p>介绍手中各种电子产品及各自的拓展玩法</p>
+<span id="more"></span>
+<h2
+id="对各种电子产品的拓展玩法与互联">对各种电子产品的拓展玩法与互联</h2>
+<p>手上有的电子产品，其中大部分应该是每个人都有的。</p>
 <ul>
-<li>C: Create Operations, 创建</li>
-<li>R: Read Operations, 读取</li>
-<li>U: Update Operations, 更新</li>
-<li>D: Delete Operations, 删除</li>
+<li><p>魅族手机</p></li>
+<li><p>笔记本电脑（Win10）</p></li>
+<li><p>1T的机械移动硬盘</p></li>
+<li><p>若干U盘。。</p></li>
+<li><p>Kindle Paper White 3</p></li>
+<li><p>树莓派Zero</p></li>
+<li><p>IPad 2018（IOS13前）</p></li>
 </ul>
-<p>mongo数据库至上而下是：<code>Database</code>--&gt;<code>Collections</code>--&gt;<code>Documents</code>。我在这里分别理解成数据库、集合、记录。</p>
-<p>这里直接用官方给的数据来操作。假设以下数据放在了<code>fruits</code>集合中。</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="number">5</span><span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">-1</span><span class="punctuation">,</span> <span class="number">3</span><span class="punctuation">]</span><span class="punctuation">,</span> <span class="attr">&quot;links&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="punctuation">&#123;</span><span class="attr">&quot;Uin&quot;</span><span class="punctuation">:</span> <span class="number">124</span><span class="punctuation">,</span> <span class="attr">&quot;NickName&quot;</span><span class="punctuation">:</span> <span class="string">&quot;123&quot;</span><span class="punctuation">&#125;</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;bananas&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="number">7</span><span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">1</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;oranges&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="attr">&quot;in stock&quot;</span><span class="punctuation">:</span> <span class="number">8</span><span class="punctuation">,</span> <span class="attr">&quot;ordered&quot;</span><span class="punctuation">:</span> <span class="number">12</span> <span class="punctuation">&#125;</span><span class="punctuation">,</span>  <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">5</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span> <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;avocados&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="string">&quot;fourteen&quot;</span> <span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">5</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
-<h3 id="基本操作">基本操作</h3>
-<p><code>json</code>数据的嵌套查找，就是说，如果有字段包含json，可以直接用<code>.</code>进行连接。</p>
-<p>查找<code>qty</code>中的<code>ordered</code>为12的字段</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty.ordered&quot;: <span class="number">12</span>&#125;)</span><br></pre></td></tr></table></figure>
-<p>多重筛选，同时满足多个要求的记录</p>
-<p>查找<code>qty</code>为5，且<code>_id</code>为"apples"的记录</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>, &quot;_id&quot;: &quot;apples&quot;&#125;)</span><br></pre></td></tr></table></figure>
-<p>第一个大括号内容表示筛选条件，第二个大括号表示显示内容</p>
-<p>筛选条件就不说明了，第二个大括号表示打印<code>_id</code>字段和<code>score</code>字段，不打印其他字段，</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>&#125;, &#123; &quot;_id&quot;: <span class="number">1</span> , &quot;score&quot;: <span class="number">1</span>&#125;)</span><br></pre></td></tr></table></figure>
-<h3 id="比较">比较</h3>
-<p>下面至上而下分别表示，在<code>fruits</code>集合中查找字段<code>qty</code>与4的大小关系所有记录。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">4</span>&#125;)          # 等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gt&quot;: <span class="number">4</span>&#125;&#125;)   # 大于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$lt&quot;: <span class="number">4</span>&#125;&#125;)   # 小于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gte&quot;: <span class="number">4</span>&#125;&#125;)  # 大于等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$lte&quot;: <span class="number">4</span>&#125;&#125;)  # 小于等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$ne&quot;: <span class="number">4</span>&#125;&#125;)   # 不等于</span><br></pre></td></tr></table></figure>
-<p>当然，也可以两个比较符一起操作，比如下面这样。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gt&quot;: <span class="number">3</span>, &quot;$lt&quot;: <span class="number">5</span>&#125;&#125;)  # 大于<span class="number">3</span>小于<span class="number">5</span>。</span><br></pre></td></tr></table></figure>
-<p>如果对<code>score</code>这个字段使用比较，比如下面这样。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;score&quot;: &#123;&quot;$gt&quot;: <span class="number">2</span>, &quot;$lt&quot;: <span class="number">5</span>&#125;&#125;)  # 大于<span class="number">3</span>小于<span class="number">5</span>。</span><br></pre></td></tr></table></figure>
-<p>将会返回下面两条记录。理由是，字段里的数组里有含有大于2小于5的数字。</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">/* 1 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;qty&quot;</span> <span class="punctuation">:</span> <span class="number">5</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">-1</span><span class="punctuation">,</span></span><br><span class="line">        <span class="number">3</span></span><br><span class="line">    <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;links&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> <span class="number">124</span><span class="punctuation">,</span></span><br><span class="line">            <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;123&quot;</span></span><br><span class="line">        <span class="punctuation">&#125;</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">/* 2 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;bananas&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;qty&quot;</span> <span class="punctuation">:</span> <span class="number">7</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">        <span class="number">5</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
-<h3 id="关系">关系</h3>
-<h4 id="in">$in</h4>
-<p>查找<code>_id</code>字段中包含<code>5</code>或者<code>ObjectId("507c35dd8fada716c89d0013")</code>的记录</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;_id&quot;: &#123;&quot;$in&quot;:[<span class="number">5</span>, ObjectId(&quot;507c35dd8fada716c89d0013&quot;)] &#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="elemmatch">$elemMatch</h4>
-<p>用于字段中含有数组，且数组中含有<code>json</code>数据的的查找。</p>
-<p>查找<code>links</code>这个含有数组的字段，找出数组中<code>Uin</code>等于124的记录</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;links&quot;: &#123;&quot;$elemMatch&quot;: &#123;&quot;Uin&quot;: <span class="number">124</span>&#125;&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="slice">$slice</h4>
-<p>切片，只打印字段中数组前n个数据</p>
-<p>筛选出记录之后，只打印记录<code>score</code>字段的前1条数据</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>&#125;, &#123;&quot;_id&quot;: <span class="number">1</span>, &quot;score&quot;: &#123;&quot;$slice&quot;: <span class="number">1</span>&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<p>输出如下</p>
-<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">/* 1 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">-1</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
-<h4 id="regex">$regex</h4>
-<p>使用正则表达式，更多操作请查看官方文档: <a
-href="https://docs.mongodb.com/manual/reference/operator/query/regex/index.html">$regex</a></p>
-<p>找出<code>_id</code>字段中所有以a开头的记录</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;_id&quot;: &#123;&quot;$regex&quot;: &quot;^a&quot;&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h3 id="后缀">后缀</h3>
-<h4 id="sort排序">sort——排序</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().sort(&#123;&quot;qty&quot;: <span class="number">1</span>&#125;) # 升序</span><br><span class="line">db.fruits.find().sort(&#123;&quot;qty&quot;: <span class="number">-1</span>&#125;) # 降序</span><br></pre></td></tr></table></figure>
-<h4 id="limit限制">limit——限制</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().limit(<span class="number">3</span>) # 对查询结果只输出前<span class="number">3</span>个，可以加在sort后面结合使用</span><br></pre></td></tr></table></figure>
-<h4 id="skip跳过">skip——跳过</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="keyword">skip</span>(<span class="number">3</span>) # 对查询结果跳过输出前<span class="number">3</span>个，即不输出前<span class="number">3</span>个，可以加在sort后面结合使用</span><br></pre></td></tr></table></figure>
-<h4 id="count计数">count——计数</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="built_in">count</span>()  # 返回查询结果总数</span><br></pre></td></tr></table></figure>
-<h4
-id="collation基于特定语言进行操作">collation——基于特定语言进行操作</h4>
-<p>没有怎么看懂，我理解的大概意思就是说，运行根据特定语言来进行查找。感觉用的不多，这里贴上官方给的例子</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="keyword">collation</span>( &#123; locale: &quot;en_US&quot;, strength: <span class="number">1</span> &#125; )</span><br></pre></td></tr></table></figure>
-<h3 id="个人常用操作">个人常用操作</h3>
-<p>例子： <figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">// test_db</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">   <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> ObjectId(<span class="string">&quot;56063f17ade2f21f36b03133&quot;</span>)<span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;title&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;MongoDB 教程&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;description&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;MongoDB 是一个 Nosql 数据库&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;by&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;菜鸟教程&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;url&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;http://www.runoob.com&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;tags&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">           <span class="string">&quot;mongodb&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="string">&quot;database&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="string">&quot;NoSQL&quot;</span></span><br><span class="line">   <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">     <span class="attr">&quot;links&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span><span class="punctuation">&#123;</span></span><br><span class="line">           <span class="attr">&quot;UserName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;@a1bf7a55ad8478017b95124774687769&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> <span class="number">12266535</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;一二三&quot;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#123;</span></span><br><span class="line">           <span class="attr">&quot;UserName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;@5571e120b5d8f54f93338c76f62b9c419b818c29373dca9f331c2911a203bfbe&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> NumberLong(<span class="number">3379331828</span>)<span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;灵灵八&quot;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#125;</span><span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure></p>
-<h4 id="返回查找结果的数目">返回查找结果的数目</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;title&quot;: &quot;MongoDB 教程&quot;&#125;).<span class="built_in">count</span>()</span><br></pre></td></tr></table></figure>
+<h3 id="手机与ipad">手机与IPad</h3>
+<p>这两个放在一起的原因是，在IOS13之前，IPad其实差不多可以等同于大尺寸IPone。所以这里就是一个安卓手机（未ROOT）与IPone的介绍，这里的玩法也算是比较多的，下面一一介绍。关于APP的安装包在此就不提供了，请读者自行搜索。当然还有ADB这种玩法，也不做介绍，可参考
+<a href="#">Post not found: IOT/用ADB调试安卓手机.md</a></p>
+<h4 id="termux与juicessh">Termux与JuiceSSH</h4>
+<p>放在搜索引擎一搜，其实这两个APP就有很多相关介绍了。但这里我想介绍一些不一样的玩法。关于Termux操作不方便这一点，可以使用JuiceSSH连接localhost来操作，具体操作不介绍。</p>
+<p><strong>下载</strong>：如何在手机上实现多线程下载？下一个ADM？当然这样很方便，但我们同样可以使用Termux来完成这一点。Termux安装Axel（<code>pkg install axel</code>），下载<code>axel -n 32 url</code>，使用32个线程下载链接。建议先<code>cd /sdcard/</code>中。</p>
+<p><strong>数据传输</strong>：一个手机和一个IPad，如何在不借助其他介质的前提下快速传输。QQ、微信、快牙等传输文件工具都行，但这里我自己实测速度都不如这里的方法快。就拿快牙举例，它实际是用手机开放一个<code>2.4G</code>的热点，然后用二维码进行链接传输。同样的，我们用同样的思路，手机开放<code>5G</code>热点，IPad手动连接此WIFI。打开Termux，先<code>ifconfig</code>知道本地IP，形如<code>192.168.x.1</code>这种，再<code>cd</code>到传输文件的目录，利用Python的静态资源服务器<code>python3 -m http.server</code>，这样，资源就对局域网内设备（连接到手机热点的设备）开放。接下来操作IPad，由于IOS13之前，<code>Safari</code>不支持输入链接下载，所以这里推荐<code>Alook</code>与<code>Documents</code>（后者免费），在<code>Documents</code>中的浏览器输入刚刚的手机<code>ip:8000</code>，这样就能看到手机资源所在文件目录的所有文件，点击即可下载。实测单个文件速度大概比<code>2.4G</code>快3倍（无需流量）。当然这里手机和IPad也可以同时连接同一个WIFI，然后使用同上方法进行传输（速度取决于路由器和网络带宽）。</p>
+<p><strong>远程手机</strong>：如何在手机不连接WIFI，不连接数据线的前提下，用其他设备访问手机（手机与其他设备的实际距离也可以很远），前提手机能上网。这一方法个人暂没有特别用途，但<code>Just for fun</code>。首先，需要一个公网IP服务器；接下来需要会玩ssh，<code>Termux</code>与服务器配置好ssh，<code>Termux</code>能够ssh到服务器上即可。最后一步，参考
+<a href="/hexo_blog/2018/02/11/%E8%BF%90%E7%BB%B4/ssh/" title="ssh 笔记（端口转发）">ssh 笔记（端口转发）</a>。</p>
+<h4 id="spacedesk">spacedesk</h4>
+<p>电脑投屏，使用USB数据线连接设备和电脑，用这个软件进行投屏。我觉得挺鸡肋的。。。安卓手机屏幕又太小，测试IPad的时候发现闪退。</p>
 <h4
-id="查找某个key中的数组是否某包含某内容">查找某个key中的数组是否某包含某内容</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;tags&quot;: &#123;&quot;$in&quot;: [&quot;mongodb&quot;]&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="使用正则进行查找">使用正则进行查找</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;url&quot;: &#123;&quot;$regex&quot;: &quot;^http&quot;&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h4 id="查找某个由数组构成的key方法">查找某个由数组构成的key方法</h4>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;links&quot;:&#123;&quot;$elemMatch&quot;:&#123;&quot;Uin&quot;:<span class="number">12266535</span>,&quot;NickName&quot;:&quot;一二三&quot;&#125;&#125;&#125;)</span><br></pre></td></tr></table></figure>
-<h2 id="附录">附录</h2>
-<h3 id="数据库权限说明表">数据库权限说明表</h3>
+id="androidhttpcapture和httpcanary">AndroidHttpCapture和HttpCanary</h4>
+<p>安卓抓包工具，后者的功能比前者更强大一些。有时候身边没有电脑，需要手机抓包看数据，所以就只能直接用手机看数据包了。前者基本满足了大部分场景的抓包数据。但当某些时候（比如某些APP的数据交互不是使用HTTP等常用协议，可能就没办法抓包，此时电脑上没办法直接抓包），所以就需要使用<code>HttpCanary</code>（需要ROOT）来对网络进行代理，在解析数据。测试如果电脑上使用<code>WireShark</code>，确实是可以抓到包的，但是会乱码。所以可能使用<code>HttpCanary</code>还更为方便一些。</p>
+<h4 id="netx与net-analyzer">NetX与Net Analyzer</h4>
+<p>安卓局域网设备分析工具，用的不多，具体不介绍，有兴趣自行下载、</p>
+<h4 id="ios同类软件免费">IOS同类软件，免费</h4>
+<ul>
+<li><p><code>iTerminal</code>，同<code>JuiceSSH</code></p></li>
+<li><p><code>Stream</code>，同<code>AndroidHttpCapture</code></p></li>
+<li><p><code>Fing</code>，同<code>NetX</code></p></li>
+</ul>
+<h4 id="scrcpy">scrcpy</h4>
+<p>在电脑上使用，安卓投屏工具，<a
+href="https://github.com/Genymobile/scrcpy">scrcpy</a>，由于魅族手机一些独特的操作，所以需要替换jar文件，<a
+href="https://github.com/taie02/taie02.github.io/tree/master/2019/03/03/scrcpy_usage">此项目</a>。</p>
+<h4 id="otg">OTG</h4>
+<p>关于OTG这一项，其实是属于外接设备这一项，通过充电接口转USB接口可以接大部分usb设备，比如u盘这种。甚至还可以反向充电，接鼠标键盘等。</p>
+<h3 id="u盘与移动硬盘">U盘与移动硬盘</h3>
+<p>二者都属于移动存储介质，其实也没什么好介绍的。。。可以提几点，Windows
+To
+Go（WTG）可以搞（高性能的移动盘），自己没尝试过。在移动硬盘上装了两个Linux系统。</p>
+<p>关于如何装系统就不介绍了，这里只有一点需要注意的是备份好数据再分区，在装系统的时候引导装载移动硬盘上，要不然移动硬盘上只能在装系统的设备上启动。</p>
+<h3 id="kindle">Kindle</h3>
+<p>这里不介绍越狱之类的操作，只有常规操作。。。</p>
+<ul>
+<li><p><a
+href="https://bookfere.com/post/19.html">搭建个人推送网站</a></p></li>
+<li><p><a href="https://volmoe.com/">漫画</a></p></li>
+<li><p>Chrome插件：Push to Kindle</p></li>
+</ul>
+<h3 id="树莓派">树莓派</h3>
+<p>吃灰派。。。大概也就能跑跑小demo，爬数据，利用空闲网络带宽下东西。同样也可以利用ssh，进行端口转发，远程连接。</p>
+<p>后续再更。。。</p>
+]]></content>
+      <categories>
+        <category>Life</category>
+      </categories>
+      <tags>
+        <tag>Raspberry</tag>
+        <tag>Linux</tag>
+        <tag>Android</tag>
+        <tag>Windows</tag>
+        <tag>IOS</tag>
+        <tag>Kindle</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>用ADB调试安卓手机</title>
+    <url>/hexo_blog/2018/01/25/IOT/%E7%94%A8ADB%E8%B0%83%E8%AF%95%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA/</url>
+    <content><![CDATA[<p>使用adb工具实现对安卓手机的调试，包括安装应用、查看运行App、录屏、截图等功能。本文详细介绍了adb工具的安装、配置和使用方法，并列出了一些实用的命令，同时还介绍了无线网调试设备的方法。</p>
+<span id="more"></span>
+<p>写在开头，一些推荐阅读的博客文章</p>
+<p><a
+href="http://blog.csdn.net/fasfaf454/article/details/51438743?locationNum=14">通过adb获取手机信息</a></p>
+<p><a href="https://www.jianshu.com/p/5980c8c282ef">Android
+adb你真的会用吗?</a></p>
+<p><a
+href="http://blog.csdn.net/qq_15364915/article/details/52369266">ADB调试命令大全</a></p>
+<h3 id="介绍">介绍</h3>
+<p>adb是安卓开发调试工具，接触到这个还是因为微信跳一跳的外挂原因。看文档说明之后得知需要adb工具，进行简单尝试后，发现这玩意还真的是挺好用的。填补了自己用代码控制安卓手机的空缺（之前也玩过按键精灵，但是需要Root权限）。</p>
+<p><a
+href="https://developer.android.com/studio/command-line/adb.html?hl=zh-cn"><strong>adb官方说明</strong></a>：</p>
+<p>Android 调试桥 (adb)
+是一个通用命令行工具，其允许您与模拟器实例或连接的 Android
+设备进行通信。它可为各种设备操作提供便利，如安装和调试应用，并提供对
+Unix
+shell（可用来在模拟器或连接的设备上运行各种命令）的访问。该工具作为一个客户端-服务器程序，包括三个组件：</p>
+<h3 id="安装">安装</h3>
+<p>可参考文章: <a
+href="https://www.xda-developers.com/install-adb-windows-macos-linux/">How
+to Install ADB on Windows, macOS, and Linux</a></p>
+<h4 id="windows">Windows</h4>
+<p>这里我使用的是win10。如果以下操作不能成功配置，可以百度其他的安装教程</p>
+<ol type="1">
+<li><p>下载。<a
+href="https://dl.google.com/android/repository/platform-tools-latest-windows.zip">下载链接</a></p></li>
+<li><p>移动目录</p>
+<ol type="1">
+<li>解压压缩包</li>
+<li>移动文件夹你想让它放置的位置</li>
+</ol></li>
+<li><p>环境配置</p>
+<ol type="1">
+<li>添加环境变量，这样可以在cmd或者powershell中直接使用adb</li>
+<li>右键"计算机"-&gt;"属性"-&gt;左边的"高级系统设置"-&gt;"环境变量"&gt;-在上面用户变量中的"Path"进行编辑-&gt;"添加"刚刚放置的文件夹目录，确认保存即可
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135305187.png" /></li>
+<li>这样再打开cmd或者powershell，就可以直接使用adb命令了</li>
+</ol></li>
+</ol>
+<h4 id="linux">Linux</h4>
+<p>Linux系统安装adb工具还是很容易的，我使用的是Ubuntu17.10</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> apt updagte</span><br><span class="line">&gt; <span class="built_in">sudo</span> apt upgrade</span><br><span class="line">&gt; <span class="built_in">sudo</span> apt install android-tools-adb</span><br></pre></td></tr></table></figure>
+<h3 id="配置">配置</h3>
+<p>手机需要打开USB调试模式，在实验的时候，市面上流行的大部分安卓手机USB调式模式开关默认关闭。当然如果手机默认开启了USB调试开关，自然可以省去下面一步。这里简单介绍一下如何手动打开USB调试模式开关，手机“设置”—&gt;"关于手机"—&gt;点击"版本号"7次。每个手机开启的方式大同小异，如果有不同，可以百度“xx手机如何开启USB调试”。</p>
+<p>在USB调试开关开启之后，进入USB调试，打开USB调试开关，如下图。</p>
+<p><img src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135319543.png" width="300px" /></p>
+<p>之后使用USB数据线连接到电脑。</p>
+<p>连接成功后，手机需要信任电脑。电脑打开cmd/powershell（Windows）或者终端（Linux），输入
+<code>adb devices</code>。查看输出。</p>
+<p>如果有输出设备名且无警告，则表示连接成功，否则需要进行下面一步的配置。</p>
+<p><strong>Windows</strong></p>
+<p>首先需要连接手机，确保手机能连接到电脑且打开了USB调试。</p>
+<p>第二步，右键"计算机"-&gt;"属性"-&gt;左边的"设备管理器"-&gt;"便携设备"-&gt;"详细信息"-&gt;"属性"下拉选择"硬件Id"，查看"值"中的内容。</p>
+<p>第三步，进入C盘用户目录下的隐藏文件夹".android"，编辑或者新建文件"adb_usb.ini"，用记事本打开，在里面写上内容。我这里需要写的是"0x2A45"，0x是前缀不需要更改，后面的字符串对应的更改"硬件Id"里面"值"的VID后面的四个字符。保存退出即可</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/adb/20180125135348748.png" /></p>
+<p>第四步，断开USB连接，重新连接，再输入
+<code>adb devices</code>。正常情况下应该会出现设备的成功连接的信息</p>
+<p><strong>Linux</strong></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 先连接手机，输入下面命令，找到关于连接手机的那一行</span></span><br><span class="line">&gt; lsusb</span><br><span class="line"><span class="comment"># 输出形如下面，最后的&quot;Google Inc&quot;表示的是设备名，这里需要找到匹配自己手机的设备，并记录好ID</span></span><br><span class="line">...</span><br><span class="line">Bus 001 Device 017: ID 18d1:d002 Google Inc.</span><br><span class="line">...</span><br><span class="line"><span class="comment"># 在目录/etc/udev/rules.d/下，添加文件70-android.rules(名字貌似无所谓？？？)</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> vim /etc/udev/rules.d/70-android.rules</span><br><span class="line"><span class="comment"># 写入下面内容，idVendor表示ID冒号前的数字，idProduct表示的是ID冒号后的数字，MODE固定为&quot;0666&quot;，其他不变</span></span><br><span class="line">SUBSYSTEM==<span class="string">&quot;usb&quot;</span>, ATTRS&#123;idVendor&#125;==<span class="string">&quot;18d1&quot;</span>, ATTRS&#123;idProduct&#125;==<span class="string">&quot;d002&quot;</span>,MODE=<span class="string">&quot;0666&quot;</span></span><br><span class="line"><span class="comment"># 保存退出，修改权限</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> <span class="built_in">chmod</span> 644   /etc/udev/rules.d/70-android.rules</span><br><span class="line">&gt; <span class="built_in">sudo</span> <span class="built_in">chown</span> root. /etc/udev/rules.d/70-android.rules</span><br><span class="line"><span class="comment"># 重启服务</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> service udev restart</span><br><span class="line">&gt; <span class="built_in">sudo</span> killall adb</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输入adb devices查看输出，是否有误</span></span><br><span class="line">&gt; adb devices</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h3 id="使用">使用</h3>
+<p>详细可参考这篇博客: <a
+href="http://blog.csdn.net/qq_15364915/article/details/52369266">ADB调试命令大全</a></p>
+<p>在这里贴出一些我认为比较有意思且实用的功能</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看所有连接设备</span></span><br><span class="line">&gt; adb devices</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行截图保存在sd卡的根目录下，名字为screen.png</span></span><br><span class="line">&gt; adb shell screencap -p /sdcard/screen.png</span><br><span class="line"><span class="comment"># 将截图发送到本地（当前目录下），也可以发送其他文件</span></span><br><span class="line">&gt; adb pull /sdcard/screen.png</span><br><span class="line"><span class="comment"># 删除本地文件</span></span><br><span class="line">&gt; adb shell <span class="built_in">rm</span> /sdcard/screen.png</span><br><span class="line"><span class="comment"># 发送电脑里的文件到设备</span></span><br><span class="line">&gt; adb shell push screen.png /sdcard/</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入手机的交互环境，操作类似linux终端，exit或者Ctrl+C退出</span></span><br><span class="line">&gt; adb shell</span><br><span class="line"></span><br><span class="line"><span class="comment"># 点击手机屏幕(1000,1000)的位置</span></span><br><span class="line">&gt; adb shell input tap 1000 1000</span><br><span class="line"><span class="comment"># 输入字符串&quot;helloworld&quot;，此处不能直接输入中文，且字符串不能有空格</span></span><br><span class="line">&gt; adb shell input text helloworld</span><br><span class="line"><span class="comment"># 滑动屏幕，从（100, 100)到（1000,1000)，经历10s(也可以当作长按屏幕来使用)</span></span><br><span class="line">&gt; adb shell input swipe 100 100 1000 1000 10</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看当前运行的App, 这里Windows没有grep所以会运行失败，可以进入先进入交互环境再输入下面去掉&quot;adb shell&quot;命令</span></span><br><span class="line">&gt; adb shell dumpsys window | grep mCurrentFocus</span><br><span class="line"><span class="comment"># 或者</span></span><br><span class="line">&gt; adb shell dumpsys activity activities | grep mFocusedActivity</span><br><span class="line"></span><br><span class="line"><span class="comment"># 按下电源键</span></span><br><span class="line">&gt; adb shell input keyevent 26</span><br><span class="line"><span class="comment"># 按下返回键</span></span><br><span class="line">&gt; adb shell input keyevent 4</span><br><span class="line"><span class="comment"># 按下HOME健</span></span><br><span class="line">&gt; adb shell input keyevent 3</span><br><span class="line"><span class="comment"># 点亮屏幕</span></span><br><span class="line">&gt; adb shell input keyevent 224</span><br><span class="line"><span class="comment"># 熄灭屏幕</span></span><br><span class="line">&gt; adb shell input keyevent 223</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看手机安装了哪些App，输出按行输出App的包名</span></span><br><span class="line">&gt; adb shell pm list packages</span><br><span class="line"><span class="comment"># 加&quot;-s&quot;表示只输出系统应用</span></span><br><span class="line"><span class="comment"># 加&quot;-3&quot;表示只输出第三方应用</span></span><br><span class="line"><span class="comment"># 加字符串表示过滤应用名称，当然也可以使用grep</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装apk</span></span><br><span class="line">&gt; adb install &lt;packagename&gt;</span><br><span class="line"><span class="comment"># 卸载apk</span></span><br><span class="line">&gt; adb uninstall &lt;packagename&gt;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 从桌面启动app</span></span><br><span class="line">&gt; adb shell monkey -p &lt;packagename&gt; -c android.intent.category.LAUNCHER 1</span><br><span class="line"><span class="comment"># 关闭app</span></span><br><span class="line">&gt; adb shell am force-stop &lt;packagename&gt;</span><br></pre></td></tr></table></figure>
+<h3 id="使用无线网调试设备">使用无线网调试设备</h3>
+<p>确保手机和电脑在同一局域网内</p>
+<p>如何查看手机的ip，由于每个设备的方法不大相同，在此不进行说明，请读者自行百度。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用USB连接设备，启动手机5555端口</span></span><br><span class="line">&gt; adb tcpip 5555</span><br><span class="line"><span class="comment"># adb连接命令，需要知道手机的ip</span></span><br><span class="line">&gt; adb connect 192.168.1.111</span><br><span class="line"><span class="comment"># 输入如下则表示连接成功</span></span><br><span class="line">connected to 192.168.1.111:5555</span><br><span class="line"><span class="comment"># 这里要注意的是，如果接下来的步骤未断掉数据线，则电脑是连接了两台设备的，进行操作会产生错误，所以可以拔掉数据线或者断开网络或者指定设备运行</span></span><br><span class="line"><span class="comment"># 断开无线网连接</span></span><br><span class="line">&gt; adb disconnect 192.168.1.111</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Android</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>ADB</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Matplotlib笔记</title>
+    <url>/hexo_blog/2017/08/26/PythonPackages/Matplotlib%E7%AC%94%E8%AE%B0/</url>
+    <content><![CDATA[<p>解决Python中文乱码问题的完整方案及Matplotlib绘图基础操作</p>
+<span id="more"></span>
+<h1 id="中文乱码解决方案">中文乱码解决方案</h1>
+<p>支持所有平台的解决方案</p>
+<h2 id="定位默认字体位置">定位默认字体位置</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先进入python的交互环境</span></span><br><span class="line">&gt;&gt;&gt;python</span><br><span class="line"></span><br><span class="line">&gt;from matplotlib.font_manager import findfont, FontProperties</span><br><span class="line"><span class="comment"># 当前使用的默认字体</span></span><br><span class="line">&gt;findfont(FontProperties(family=FontProperties().get_family()))</span><br><span class="line"><span class="comment"># 在此记住返回值</span></span><br></pre></td></tr></table></figure>
+<h2 id="下载支持中文的字体">下载支持中文的字体</h2>
+<p>在这里推荐微软雅黑，msyh.ttf。</p>
+<p><a
+href="http://www.monmonkey.com/sonota/font1/getfont.html">推荐链接</a></p>
+<h2 id="偷梁换柱">偷梁换柱</h2>
+<p>在刚刚查询的那个文件夹中，将原来的默认字体重命名，之后将我们下载msyh.ttf替换进去，并重命名为那个默认字体之前的名字</p>
+<h2 id="大功告成">大功告成</h2>
+<p>当然为了保险起见，还可以操作下面一步。到现在为止，可以测试看看是否成功输出中文。</p>
+<p>找到matplotlib的安装目录，一般是在python的安装目录下的-packages-data,之后修改matplotlibrc文件</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 将下面这两行冒号后面的字符替换</span></span><br><span class="line">...</span><br><span class="line"><span class="comment">#font.sans-serif     : nothing</span></span><br><span class="line">...</span><br><span class="line"><span class="comment">#verbose.level  : debug</span></span><br></pre></td></tr></table></figure>
+<h1 id="matplotlib.pyplot-基础操作">matplotlib.pyplot 基础操作</h1>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> matplotlib.pyplot <span class="keyword">as</span> plt</span><br><span class="line"></span><br><span class="line">x_list = []</span><br><span class="line">y_list = []</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定画布</span></span><br><span class="line">plt.figure(figsize=(<span class="number">8</span>, <span class="number">6</span>), dpi=<span class="number">80</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 画y_list的折线图，格式为&#x27;ro-&#x27;，颜色为blue</span></span><br><span class="line">plt.plot(y_list, <span class="string">&#x27;ro-&#x27;</span>, color=<span class="string">&#x27;blue&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以x_list为x轴，y_list为y轴画折线图，前提x_list内容全为整型数字</span></span><br><span class="line">plt.plot(x_list, y_list)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以x_list为x轴，y_list为y轴，x_list为字符串</span></span><br><span class="line">x = <span class="built_in">range</span>(<span class="built_in">len</span>(x_list))</span><br><span class="line">plt.plot(x, y_list)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定横坐标, 跟着上面的字符串x_list的设定，每个字符串旋转角度为45，字体大小为20</span></span><br><span class="line">plt.xticks(x, x_list, rotation=<span class="number">45</span>, fontsize=<span class="number">20</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 设定标题，注如果需要设定中文需要更改matplotlib的字体配置</span></span><br><span class="line">plt.title(<span class="string">&#x27;title&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 保存图片名为plt</span></span><br><span class="line">plt.savefig(<span class="string">&quot;plt.png&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 运行完打开图片</span></span><br><span class="line">plt.show()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 注：如果需要保存图片，切记需要在打开图片前执行命令，否则保存图片为空白</span></span><br></pre></td></tr></table></figure>
+<h1 id="搬运工系列">搬运工系列</h1>
+<p><a
+href="https://juejin.im/post/5a9e14726fb9a028b86d87c9">5种快速易用的Python
+Matplotlib数据可视化方法</a></p>
+<p><a
+href="http://www.labri.fr/perso/nrougier/teaching/matplotlib/">Matplotlib
+教程</a></p>
+<p><a
+href="https://liam0205.me/2014/09/11/matplotlib-tutorial-zh-cn/">Matplotlib
+教程（中文翻译版本）</a></p>
+]]></content>
+      <categories>
+        <category>Visualization</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>Matplotlib</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>优化pandas内存入门</title>
+    <url>/hexo_blog/2018/03/17/PythonPackages/%E4%BC%98%E5%8C%96pandas%E5%86%85%E5%AD%98%E5%85%A5%E9%97%A8/</url>
+    <content><![CDATA[<p>降低数据存储消耗并提高处理效率，本文教你如何通过优化数值类型、<code>object</code>类型和
+<code>datetime</code>类型三个方面来减少内存使用量。其中，使用
+<code>Categoricals</code>优化
+<code>object</code>类型数据是效果最显著的一种优化方式。另外，还介绍了如何在读取数据的时候就完成优化步骤，避免加载完数据再做优化的麻烦。</p>
+<span id="more"></span>
+<h2 id="简单的概念">简单的概念</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"><span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line"></span><br><span class="line"><span class="comment"># 读取数据</span></span><br><span class="line">df = pd.read_csv(<span class="string">&quot;test.csv&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 得到精确的内存信息</span></span><br><span class="line">df.info(memory_usage=<span class="string">&#x27;deep&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 说明：之后都用df来表示读取到的dataframe</span></span><br></pre></td></tr></table></figure>
+<p>pandas中每一个数据类型都有一个专门的类来处理。</p>
+<ul>
+<li><code>ObjectBlock</code>： 字符串列的块</li>
+<li><code>FloatBlock</code>： 浮点数列的块</li>
+<li><code>Numpy ndarray</code>：整型和浮点数值的块（非常快，用C数组构建的）</li>
+</ul>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> dtype <span class="keyword">in</span> [<span class="string">&#x27;float&#x27;</span>, <span class="string">&#x27;int&#x27;</span>, <span class="string">&#x27;object&#x27;</span>]:</span><br><span class="line">    <span class="comment"># 选中对应的dtype列</span></span><br><span class="line">    selected_dtype = df.select_dtypes(include=[dtype])</span><br><span class="line">    <span class="comment"># 查看内存使用量的平均值</span></span><br><span class="line">    mean_usage_b = selected_dtype.memory_usage(deep=<span class="literal">True</span>).mean()</span><br><span class="line">    <span class="comment"># 获取到的数据单位为K, 这里转换一下</span></span><br><span class="line">    mean_usage_mb = mean_usage_b / <span class="number">1024</span> ** <span class="number">2</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;Average memory usage for &#123;&#125; columns:&#123;:03.2f&#125; MB&quot;</span>.<span class="built_in">format</span>(dtype, mean_usage_mb))</span><br></pre></td></tr></table></figure>
+<p>这里我们可以发现 <code>object</code>类占用内存最大</p>
+<h3 id="子类型subtype">子类型(subtype)</h3>
 <table>
-<colgroup>
-<col style="width: 23%" />
-<col style="width: 76%" />
-</colgroup>
 <thead>
 <tr>
-<th>名称</th>
-<th style="text-align: left;">权限</th>
+<th>memory usage</th>
+<th>float</th>
+<th>int</th>
+<th>uint</th>
+<th>datetime</th>
+<th>bool</th>
+<th>object</th>
 </tr>
 </thead>
 <tbody>
 <tr>
-<td>数据库用户角色</td>
-<td style="text-align: left;">read、readWrite</td>
+<td>1 bytes</td>
+<td></td>
+<td>int8</td>
+<td>uint8</td>
+<td></td>
+<td>bool</td>
+<td></td>
 </tr>
 <tr>
-<td>数据库管理角色</td>
-<td style="text-align: left;">dbAdmin、dbOwner、userAdmin</td>
+<td>2 bytes</td>
+<td>float16</td>
+<td>int16</td>
+<td>uint16</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>集群管理角色</td>
-<td
-style="text-align: left;">clusterAdmin、clusterManager、clusterMonitor、hostManager</td>
+<td>4 bytes</td>
+<td>float32</td>
+<td>int32</td>
+<td>uint32</td>
+<td></td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>备份恢复角色</td>
-<td style="text-align: left;">backup、restore</td>
-</tr>
-<tr>
-<td>所有数据库角色</td>
-<td
-style="text-align: left;">readAnyDatabase、readWriteAnyDatabase、userAdminAnyDatabase、dbAdminAnyDatabase</td>
-</tr>
-<tr>
-<td>超级用户角色</td>
-<td style="text-align: left;">root</td>
-</tr>
-<tr>
-<td>提供了系统超级用户的访问</td>
-<td style="text-align: left;">dbOwner
-、userAdmin、userAdminAnyDatabase</td>
+<td>8 bytes</td>
+<td>float64</td>
+<td>int64</td>
+<td>uint64</td>
+<td>datetime64</td>
+<td></td>
+<td></td>
 </tr>
 <tr>
-<td>内部角色</td>
-<td style="text-align: left;">__system</td>
+<td>variable</td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td></td>
+<td>object</td>
 </tr>
 </tbody>
 </table>
-<p>下一篇：<a href="/hexo_blog/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/" title="Mongodb：副本集配置">Mongodb：副本集配置</a></p>
+<p>一个
+<code>int8</code>类型值使用1个字节的存储空间，可以表示256(2^8)个二进制数，即-128到127的所有整数值。根据表可以得出，对于每个列，应该尽可能"抠"一点，
+物尽其用。这里的
+<code>uint</code>表示无符号整型，可以更有效地处理正整数的列。</p>
+<h2 id="优化">优化</h2>
+<h3 id="数值型数据">数值型数据</h3>
+<p>这里可以使用 <code>to_numeric()</code>对数值类型进行
+<code>downcast</code>（向下转型）的操作。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 选中整数型的列</span></span><br><span class="line">df_int_columns = df.select_dtypes(include=[<span class="string">&#x27;int&#x27;</span>])</span><br><span class="line"><span class="comment"># 向下转型</span></span><br><span class="line">df_int_columns.apply(pd.to_numeric, downcast=<span class="string">&#x27;unsigned&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选中浮点型数据</span></span><br><span class="line">df_float_columns = df.select_dtypes(include=[<span class="string">&#x27;float&#x27;</span>])</span><br><span class="line"><span class="comment"># 向下转型</span></span><br><span class="line">df_float_columns.apply(pd.to_numeric, downcast=<span class="string">&#x27;float&#x27;</span>)</span><br></pre></td></tr></table></figure>
+<p>这里能够有效的利用内存，但是缺点很明显的就是只能用于数值型数据，而且优化的空间有限。</p>
+<h3 id="object型数据"><code>object</code>型数据</h3>
+<p><code>object</code>类型实际上使用的是Python字符串对应的值，由于Python解释性语言的特性，字符串存储方式很碎片化（消耗更多内存，访问速度更慢）。<code>object</code>列中的每个元素都是一个指针。<code>object</code>类型的数据内存使用情况是可变的。如果每个字符串是单独存储的，那么实际上字符串占用内存是很大。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">In [<span class="number">1</span>]: <span class="keyword">from</span> sys <span class="keyword">import</span> getsizeof</span><br><span class="line"><span class="comment"># 字符串本身内存使用情况</span></span><br><span class="line">In [<span class="number">2</span>]: s1 = <span class="string">&quot;hello world!&quot;</span></span><br><span class="line">In [<span class="number">3</span>]: s2 = <span class="string">&quot;hello pandas!&quot;</span></span><br><span class="line">In [<span class="number">4</span>]: getsizeof(s1), getsizeof(s2)</span><br><span class="line">Out[<span class="number">4</span>]: (<span class="number">61</span>, <span class="number">62</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串(object)在pandas中内存使用情况</span></span><br><span class="line">In [<span class="number">9</span>]: <span class="keyword">import</span> pandas <span class="keyword">as</span> pd</span><br><span class="line">In [<span class="number">10</span>]: ser = pd.Series([s1, s2])</span><br><span class="line">In [<span class="number">11</span>]: ser.apply(getsizeof)</span><br><span class="line">Out[<span class="number">11</span>]:</span><br><span class="line"><span class="number">0</span>    <span class="number">61</span></span><br><span class="line"><span class="number">1</span>    <span class="number">62</span></span><br><span class="line">dtype: int64</span><br></pre></td></tr></table></figure>
+<p>这里通过观察看到，在这里pandas使用了int64来存储，实际占用大小与字符串本身是一样的。这里可以使用
+<code>Categoricals</code>来优化
+<code>object</code>类型。<code>Categoricals</code>的工作原理我理解为，某个
+<code>object</code>类有有限的分类情况(比如只有
+<code>red</code>、<code>yellow</code>、<code>blue</code>、<code>black</code>等颜色相关)，那么
+<code>Categoricals</code>将这些分类 <code>object</code>对象转换为
+<code>int</code>子类型（对应上面的0, 1, 2, 3）</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 选中object类型</span></span><br><span class="line">df_obj_columns = df.select_dtypes(include=[<span class="string">&#x27;object&#x27;</span>])</span><br><span class="line"><span class="comment"># 查看object类型列的相关信息</span></span><br><span class="line">df_obj_columns.describe()</span><br></pre></td></tr></table></figure>
+<p>如果
+<code>unique</code>(不同值)的数量很少，那么就可以使用这种优化方案。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 假设某一列符合优化条件，为df_obj_less，使类型转换为category</span></span><br><span class="line">df_category_column = df_obj_less.astype(<span class="string">&#x27;category&#x27;</span>)</span><br><span class="line"><span class="comment"># 转换之后，主观观察数据没有什么区别，只是类型改变了</span></span><br><span class="line"><span class="comment"># 观察转换之后实际上的数据</span></span><br><span class="line">df_category_column.cat.codes</span><br><span class="line"><span class="comment"># 变成了由0,1,2等数字构成的int8类型的数据</span></span><br></pre></td></tr></table></figure>
+<p>这里提升的空间远远超过第一步优化的空间，具体要结合数据来检验（最好貌似可以减少98%的使用量）。这里有个很大的缺点就是无法进行数值计算，即没有办法使用
+<code>pd.Series.min()</code>、<code>pd.Series.max()</code>等与数值相关操作。</p>
+<p>还有一个问题就是在有多少个
+<code>unique</code>(不同值)的情况下才使用这种方法。首先，毫无疑问的是如果需要计算操作的列是不能使用的。第二是如果
+<code>unique</code>的比例小于50%(个人觉得比例应该更小)就可以使用这种情况，如果过多的话，转换之后消耗的内存会更多（不仅需要存储string还有int）。</p>
+<p>附上筛选 <code>unique</code>比例小于50%的代码</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 提取unique少于50%的object列</span></span><br><span class="line">df_obj_columns = df.select_dtypes(include=[<span class="string">&#x27;object&#x27;</span>])</span><br><span class="line"><span class="keyword">for</span> obj_col <span class="keyword">in</span> df_obj_columns:</span><br><span class="line">    unique_values = <span class="built_in">len</span>(df_obj_columns[obj_col].unique())</span><br><span class="line">    total_values = <span class="built_in">len</span>(df_obj_columns[obj_col])</span><br><span class="line">    <span class="keyword">if</span> unique_values / total_values &lt; <span class="number">0.5</span>:</span><br><span class="line">        obj_col.astype(<span class="string">&#x27;category&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建议提前copy一份数据，再做上面的操作</span></span><br></pre></td></tr></table></figure>
+<h3 id="datetime类型"><code>datetime</code>类型</h3>
+<p>这里其实不能叫做优化，因为这里是将数值型数据转换为
+<code>datetime</code>类型，虽然提高了内存的使用，但是转换为
+<code>datetime</code>类型的数据更容易进行分析（时间序列分析）。当然如果有
+<code>datetime</code>类型不在分析的范围内，自然可以无视。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 假设df_num_col为需要转换的列, 这里format格式看情况更改</span></span><br><span class="line">df_num_col = pd.to_datetime(df_num_col, <span class="built_in">format</span>=<span class="string">&quot;%Y%m%d&quot;</span>)</span><br></pre></td></tr></table></figure>
+<h2 id="总结">总结</h2>
+<p>写了这么多，如果每次等加载完数据再做优化操作，感觉有些鸡肋。但是其实可以使用
+<code>read_csv</code>等读取函数的几个参数来帮助我们在读取的时候就完成优化步骤。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先，需要整理出每一列最终的数据类型，组成一个dict</span></span><br><span class="line">dtypes_lst = [col.name <span class="keyword">for</span> name <span class="keyword">in</span> df.dtypes]</span><br><span class="line">column_types = <span class="built_in">dict</span>(<span class="built_in">zip</span>(dtypes.index, dtypes_lst))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重新读取数据, dtype指定数据格式，parse_dates指定列转换格式为datetime,infer_datetime_format尝试解析字符串形式的datatime格式（在一些情况下可以加快解析速度5-10倍）</span></span><br><span class="line">new_df = pd.read_csv(<span class="string">&#x27;test.csv&#x27;</span>, dtpye=column_types, parse_dates=[<span class="string">&#x27;date&#x27;</span>], infer_datetime_format=<span class="literal">True</span>)</span><br></pre></td></tr></table></figure>
 ]]></content>
       <categories>
-        <category>Database</category>
+        <category>Note</category>
       </categories>
       <tags>
-        <tag>mongodb</tag>
-        <tag>数据库</tag>
+        <tag>Python</tag>
+        <tag>pandas</tag>
       </tags>
   </entry>
   <entry>
-    <title>SQL语句的优化</title>
-    <url>/hexo_blog/2018/04/07/%E6%95%B0%E6%8D%AE%E5%BA%93/SQL%E8%AF%AD%E5%8F%A5%E7%9A%84%E4%BC%98%E5%8C%96/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了关于SQL语句（主要是Oracle数据库）的优化方案。
-<span id="more"></span></p>
+    <title>Numpy的70个操作</title>
+    <url>/hexo_blog/2018/03/02/PythonPackages/Numpy%E7%9A%8470%E4%B8%AA%E6%93%8D%E4%BD%9C/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Numpy从入门到进阶的操作,
+翻译文章，便于自己使用时查阅</p>
+<span id="more"></span>
 <p><a
-href="https://juejin.im/post/5ab0f39df265da23866fba55">数据库性能优化之SQL语句优化</a></p>
-<h2 id="操作符优化">操作符优化</h2>
-<h3 id="in-操作符">IN 操作符</h3>
-<p>优点：编写容易，清晰易懂</p>
-<p>缺点：性能较低。</p>
-<p>Oracle执行步骤：</p>
-<ol type="1">
-<li>将其转换多表连接</li>
-<li>如果转换不成功则先执行IN里面的子查询，再执行外层表记录</li>
-<li>如果转换成功则直接采用多表连接查询</li>
-</ol>
-<p>所以用IN的SQL至少会多一个转换过程。</p>
-<p>替代方案：在业务密集的SQL用EXISTS替代</p>
-<h3 id="not-in操作符">NOT IN操作符</h3>
-<p>不能应用表的索引，不建议使用。</p>
-<p>替代方案：用NOT EXISTS方案</p>
-<h3 id="is-null或is-not-null操作">IS NULL或IS NOT NULL操作</h3>
-<p>判断字段为空一般是不会用索引的（索引不索引空值）。如果某列存在空值，即使对该列建索引也不会提高性能。</p>
-<p>替代方案：将<code>a is not null</code>改为<code>a&gt;0</code>或<code>a&gt;' '</code>等。不允许字段为空，而用一个缺省值代替空值。</p>
-<h3 id="及-操作符">&lt; 及 &gt; 操作符</h3>
-<p>一般情况下是不需要调整的，在某些情况下可以进行优化。</p>
-<p>如：一个表有100万记录，一个数值型字段A，有30万的A=0，39万A=2，
-1万A=3。在这种情况下执行A&gt;2与A&gt;=3有很大区别，<code>A&gt;2</code>在Oracle中会先找出为2的记录索引再进行比较，但是<code>A&gt;=3</code>会直接找到等于3的记录索引。</p>
-<h3 id="like操作符">LIKE操作符</h3>
-<p>LIKE医用通配符查询，基本上可以达到任一查询。但是不同语句可能会产生性能上的很大差异。如<code>LIKE '%5400%'</code>这种情况下不会引用索引，<code>LIKE 'X5400%'</code>会引用范围索引。即前者会产生全表扫描，后者会进行范围内的查询，后者相对于前者的性能大大提高了。</p>
-<h3 id="union操作符">UNION操作符</h3>
-<p>UNION在进行表连接之后会筛选掉重复的记录，所以在表连接之后会对产生的结果集进行排序运算，删除重复记录再返回结果。但是实际上大部分应用中是不会产生重复的记录，最常见的是过程表于历史表UNION。在这种情况下，替代方案就是使用UNION
-ALL代替UNION。UNION ALL操作只是简单的将两个结果合并后就返回</p>
-<h3 id="联接列">联接列</h3>
-<p>对于有联接的列，优化器是不会使用索引的。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 使用联接查询</span><br><span class="line"><span class="keyword">select</span> <span class="operator">*</span> <span class="keyword">from</span> employss <span class="keyword">where</span> first_name<span class="operator">||</span><span class="string">&#x27;&#x27;</span><span class="operator">||</span>last_name <span class="operator">=</span><span class="string">&#x27;Beill Cliton&#x27;</span>;</span><br><span class="line"># 不使用联接查询，基于last_name创建索引</span><br><span class="line"><span class="keyword">where</span> first_name <span class="operator">=</span><span class="string">&#x27;Beill&#x27;</span> <span class="keyword">and</span> last_name <span class="operator">=</span><span class="string">&#x27;Cliton&#x27;</span>;</span><br></pre></td></tr></table></figure>
-<h3 id="order-by语句">Order By语句</h3>
-<p>将返回的查询结果进行排序。任何在Order
-by语句的非索引项或者有计算表达式都将降低查询速度。解决方案就是为所使用的列建立另一个索引，同时也绝对避免在order
-by子句中使用表达式。</p>
-<h3 id="not-语句">NOT 语句</h3>
-<p>NOT可用来对任何逻辑运算符号取反。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 如果要使用<span class="keyword">NOT</span>，则应在取反的短语前面机上括号，并在短语前面机上<span class="keyword">NOT</span>运算符。</span><br><span class="line"><span class="keyword">where</span> <span class="keyword">not</span> (status <span class="operator">=</span> <span class="string">&#x27;VALID&#x27;</span>)</span><br><span class="line"># <span class="keyword">NOT</span>运算符包含在另外一个逻辑运算符<span class="operator">&lt;&gt;</span>运算符中。即使不在查询的<span class="keyword">where</span>子句中显式地加入<span class="keyword">NOT</span>词，<span class="keyword">NOT</span>仍在运算符中</span><br><span class="line"><span class="keyword">where</span> status <span class="operator">&lt;&gt;</span> <span class="string">&#x27;INVALID&#x27;</span>;</span><br><span class="line"># 这种情况运行建立salary的索引，在速度上会优于上面的方案</span><br><span class="line"><span class="keyword">select</span> <span class="operator">*</span> <span class="keyword">from</span> employee <span class="keyword">where</span> salary <span class="operator">&lt;</span> <span class="number">3000</span> <span class="keyword">or</span> salary <span class="operator">&gt;</span> <span class="number">3000</span>;</span><br></pre></td></tr></table></figure>
-<h2 id="书写的影响">书写的影响</h2>
-<h3
-id="同一功能同一性能不同写法sql的影响">同一功能同一性能不同写法SQL的影响</h3>
-<p>尽可能避免不同写法的出现，根据Oracle共享内存SGA原理，如果语句不同则进行分析，占用共享内存。如果SQL字符串及格式完全相同，则Oracle只会分析一次，共享内存也只会留下一次的分析结果</p>
-<h3 id="where后面的条件顺序影响">WHERE后面的条件顺序影响</h3>
-<p>下面两个语句，在没有建立索引的情况下，执行都是全表扫描。</p>
-<p><code>dy_dj = '1KV以下'</code>这个条件在记录集内比率为99%,
-<code>xh_bz=1</code>而这个条件只有0.5%。所以第二条SQL的CPU占用率明显低于第一条。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">Select</span> <span class="operator">*</span> <span class="keyword">from</span> zl_yhjbqk <span class="keyword">where</span> dy_dj <span class="operator">=</span> <span class="string">&#x27;1KV以下&#x27;</span> <span class="keyword">and</span> xh_bz<span class="operator">=</span><span class="number">1</span></span><br><span class="line"><span class="keyword">Select</span> <span class="operator">*</span> <span class="keyword">from</span> zl_yhjbqk <span class="keyword">where</span> xh_bz<span class="operator">=</span><span class="number">1</span> <span class="keyword">and</span> dy_dj <span class="operator">=</span> <span class="string">&#x27;1KV以下&#x27;</span></span><br></pre></td></tr></table></figure>
-<h3 id="查询表顺序的影响">查询表顺序的影响</h3>
-<p>在FROM后面的表中的列表顺序会对SQL执行性能影响，在没有索引及ORACLE没有对表进行统计分析的情况下，ORACLE会按<strong>表出现的顺序</strong>进行连接，所以如果表的顺序不对时会产生十分消耗服务器资源的数据交叉（如果对表进行了统计分析，Oracle自动先进小表的连接，再进行大表的连接）</p>
-<h2 id="sql语句索引的利用">SQL语句索引的利用</h2>
-<h3 id="条件字段的优化">条件字段的优化</h3>
-<p>函数处理的字段不能利用索引</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：substr(hbs_bh,<span class="number">1</span>,<span class="number">4</span>)<span class="operator">=</span><span class="string">&#x27;5400&#x27;</span></span><br><span class="line">优化处理：hbs_bh <span class="keyword">like</span> <span class="string">&#x27;5400%&#x27;</span></span><br><span class="line"></span><br><span class="line">原语句：trunc(sk_rq)<span class="operator">=</span>trunc(sysdate)</span><br><span class="line">优化处理：sk_rq<span class="operator">&gt;=</span>trunc(sysdate) <span class="keyword">and</span> sk_rq<span class="operator">&lt;</span>trunc(sysdate<span class="operator">+</span><span class="number">1</span>)</span><br></pre></td></tr></table></figure>
-<p>进行了显示或隐式的运算的字段不能进行索引。</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：<span class="string">&#x27;X&#x27;</span> <span class="operator">||</span> hbs_bh<span class="operator">&gt;</span><span class="string">&#x27;X5400021452&#x27;</span></span><br><span class="line">优化处理：hbs_bh<span class="operator">&gt;</span><span class="string">&#x27;5400021542&#x27;</span></span><br><span class="line"></span><br><span class="line">原语句：sk_rq<span class="operator">+</span><span class="number">5</span><span class="operator">=</span>sysdate</span><br><span class="line">优化处理：sk_rq<span class="operator">=</span>sysdate<span class="number">-5</span></span><br></pre></td></tr></table></figure>
-<p>条件内包括的多个本表的字段运算时不能进行索引</p>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：ys_df<span class="operator">&gt;</span>cx_df</span><br><span class="line">无法进行优化</span><br><span class="line"></span><br><span class="line">原语句：qc_bh <span class="operator">||</span> kh_bh<span class="operator">=</span><span class="string">&#x27;5400250000&#x27;</span></span><br><span class="line">优化处理：qc_bh<span class="operator">=</span><span class="string">&#x27;5400&#x27;</span> <span class="keyword">and</span> kh_bh<span class="operator">=</span><span class="string">&#x27;250000&#x27;</span></span><br></pre></td></tr></table></figure>
-<h2 id="其他优化方法">其他优化方法</h2>
+href="https://www.machinelearningplus.com/101-numpy-exercises-python/">原文链接</a></p>
+<p>注释及输出前带有 <code>#</code></p>
+<h2 id="section">1-10</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看numpy的版本</span></span><br><span class="line">np.__version__</span><br><span class="line"><span class="comment"># 1.13.3</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个一维数据</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个布尔型数组</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.full((<span class="number">3</span>, <span class="number">3</span>), <span class="literal">True</span>, dtype=<span class="built_in">bool</span>)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.ones((<span class="number">3</span>, <span class="number">3</span>), dtype=<span class="built_in">bool</span>)</span><br><span class="line"><span class="comment"># array([[ True,  True,  True],</span></span><br><span class="line"><span class="comment">#        [ True,  True,  True],</span></span><br><span class="line"><span class="comment">#        [ True,  True,  True]], dtype=bool)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取数组中为奇数的数字</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line">arr[arr % <span class="number">2</span> == <span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([1, 3, 5, 7, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 替换数组中为奇数的数字为-1</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">arr[arr % <span class="number">2</span> == <span class="number">1</span>] = -<span class="number">1</span></span><br><span class="line"><span class="comment"># 方法二, 更加安全的操作方法（这里算新的一个操作）：</span></span><br><span class="line">out = np.where(arr % <span class="number">2</span> == <span class="number">1</span>, -<span class="number">1</span>, arr)</span><br><span class="line">out</span><br><span class="line"><span class="comment"># array([ 0, -1,  2, -1,  4, -1,  6, -1,  8, -1])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数组维度变换</span></span><br><span class="line">arr = np.arange(<span class="number">10</span>) <span class="comment"># 原来的数组</span></span><br><span class="line">arr.reshape(<span class="number">2</span>, -<span class="number">1</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 纵向堆叠两个数组</span></span><br><span class="line">a = np.arange(<span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.repeat(<span class="number">1</span>, <span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组b</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.concatenate([a, b], axis=<span class="number">0</span>)</span><br><span class="line"><span class="comment"># 方法二:</span></span><br><span class="line">np.vstack([a, b])</span><br><span class="line"><span class="comment"># 方法三：</span></span><br><span class="line">np.r_[a, b]</span><br><span class="line"><span class="comment"># 三种方法的输出</span></span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9],</span></span><br><span class="line"><span class="comment">#        [1, 1, 1, 1, 1],</span></span><br><span class="line"><span class="comment">#        [1, 1, 1, 1, 1]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 横向堆叠两个数组</span></span><br><span class="line">a = np.arange(<span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.repeat(<span class="number">1</span>, <span class="number">10</span>).reshape(<span class="number">2</span>, -<span class="number">1</span>) <span class="comment"># 原来的数组b</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.concatenate([a, b], axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 方法二:</span></span><br><span class="line">np.hstack([a, b])</span><br><span class="line"><span class="comment"># 方法三：</span></span><br><span class="line">np.c_[a, b]</span><br><span class="line"><span class="comment"># 三种方法的输出</span></span><br><span class="line"><span class="comment"># array([[0, 1, 2, 3, 4, 1, 1, 1, 1, 1],</span></span><br><span class="line"><span class="comment">#        [5, 6, 7, 8, 9, 1, 1, 1, 1, 1]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据原有的数组，自由拓展</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]) <span class="comment"># 原来的数组</span></span><br><span class="line">np.r_[np.repeat(a, <span class="number">3</span>), np.tile(a, <span class="number">3</span>)]</span><br><span class="line"><span class="comment"># array([1, 1, 1, 2, 2, 2, 3, 3, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3])</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-1">11-20</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取两个数组中公共的元素</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])  <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">7</span>, <span class="number">2</span>, <span class="number">10</span>, <span class="number">2</span>, <span class="number">7</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">8</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">np.intersect1d(a, b)</span><br><span class="line"><span class="comment"># array([2, 4])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在a数组中移去a、b数组中公共的元素</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>])	<span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">9</span>])	<span class="comment"># 原来的数组b</span></span><br><span class="line">np.setdiff1d(a, b)</span><br><span class="line"><span class="comment"># array([1, 2, 3, 4])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取两个数组中公共元素（索引相同）的索引</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])  <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">7</span>, <span class="number">2</span>, <span class="number">10</span>, <span class="number">2</span>, <span class="number">7</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">4</span>, <span class="number">9</span>, <span class="number">8</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">np.where(a == b)</span><br><span class="line"><span class="comment"># (array([1, 3, 5, 7], dtype=int64),)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取指定数值区间的元素</span></span><br><span class="line">a = np.arange(<span class="number">15</span>) <span class="comment"># 原来的数组</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">index = np.where((a &gt;= <span class="number">5</span>) &amp; (a &lt;= <span class="number">10</span>))</span><br><span class="line">a[index]</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">index = np.where(np.logical_and(a&gt;=<span class="number">5</span>, a&lt;=<span class="number">10</span>))</span><br><span class="line">a[index]</span><br><span class="line"><span class="comment"># 方法三</span></span><br><span class="line">a[(a&gt;= <span class="number">5</span>) &amp; (a &lt;= <span class="number">10</span>)]</span><br><span class="line"><span class="comment"># array([ 5,  6,  7,  8,  9, 10])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># numpy运用函数</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">maxx</span>(<span class="params">x, y</span>):		<span class="comment"># 需要使用的函数</span></span><br><span class="line">    <span class="keyword">if</span> x &gt;= y:</span><br><span class="line">        <span class="keyword">return</span> x</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        <span class="keyword">return</span> y</span><br><span class="line"></span><br><span class="line">a = np.array([<span class="number">5</span>, <span class="number">7</span>, <span class="number">9</span>, <span class="number">8</span>, <span class="number">6</span>, <span class="number">4</span>, <span class="number">5</span>]) <span class="comment"># 原来的数组a</span></span><br><span class="line">b = np.array([<span class="number">6</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">8</span>, <span class="number">9</span>, <span class="number">7</span>, <span class="number">1</span>]) <span class="comment"># 原来的数组b</span></span><br><span class="line">pair_max = np.vectorize(maxx, dtypes=[<span class="built_in">float</span>])</span><br><span class="line">pair_max(a, b)</span><br><span class="line"><span class="comment"># array([ 6.,  7.,  9.,  8.,  9.,  7.,  5.])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组交换两行</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)	<span class="comment"># 原来的数组</span></span><br><span class="line">arr[:, [<span class="number">1</span>, <span class="number">0</span>, <span class="number">2</span>]]</span><br><span class="line"><span class="comment"># array([[1, 0, 2],</span></span><br><span class="line"><span class="comment">#        [4, 3, 5],</span></span><br><span class="line"><span class="comment">#        [7, 6, 8]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组交换两列</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)	<span class="comment"># 原来的数组</span></span><br><span class="line">arr[[<span class="number">1</span>, <span class="number">0</span>, <span class="number">2</span>], :]</span><br><span class="line"><span class="comment"># array([[3, 4, 5],</span></span><br><span class="line"><span class="comment">#        [0, 1, 2],</span></span><br><span class="line"><span class="comment">#        [6, 7, 8]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组每行逆序输出</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)  <span class="comment"># 原来的数组</span></span><br><span class="line">arr[::-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([[6, 7, 8],</span></span><br><span class="line"><span class="comment">#        [3, 4, 5],</span></span><br><span class="line"><span class="comment">#        [0, 1, 2]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组每列逆序输出</span></span><br><span class="line">arr = np.arange(<span class="number">9</span>).reshape(<span class="number">3</span>,<span class="number">3</span>)  <span class="comment"># 原来的数组</span></span><br><span class="line">arr[::-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># array([[2, 1, 0],</span></span><br><span class="line"><span class="comment">#        [5, 4, 3],</span></span><br><span class="line"><span class="comment">#        [8, 7, 6]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个二维数组，数值区间为5-10</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">rand_arr = np.random.randint(low=<span class="number">5</span>, high=<span class="number">10</span>, size=(<span class="number">5</span>,<span class="number">3</span>)) + np.random.random((<span class="number">5</span>,<span class="number">3</span>))</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">rand_arr = np.random.uniform(<span class="number">5</span>,<span class="number">10</span>, size=(<span class="number">5</span>,<span class="number">3</span>))</span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[ 6.75709583,  7.4662972 ,  7.2395679 ],</span></span><br><span class="line"><span class="comment">#        [ 9.7802112 ,  6.70830431,  6.76776651],</span></span><br><span class="line"><span class="comment">#        [ 6.35564151,  7.8632512 ,  8.53040631],</span></span><br><span class="line"><span class="comment">#        [ 7.99457763,  7.85526309,  8.96584465],</span></span><br><span class="line"><span class="comment">#        [ 6.18621991,  8.14214732,  7.31162037]])</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-2">21-30</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出数组时，控制输出精度</span></span><br><span class="line">rand_arr = np.random.random([<span class="number">3</span>,<span class="number">3</span>]) / <span class="number">1e3</span>  <span class="comment"># 原始数组</span></span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[  5.43404942e-04,   2.78369385e-04,   4.24517591e-04],</span></span><br><span class="line"><span class="comment">#        [  8.44776132e-04,   4.71885619e-06,   1.21569121e-04],</span></span><br><span class="line"><span class="comment">#        [  6.70749085e-04,   8.25852755e-04,   1.36706590e-04]])</span></span><br><span class="line"><span class="comment"># 第一个操作，指定小数点后3位数</span></span><br><span class="line">np.set_printoptions(precision=<span class="number">3</span>)</span><br><span class="line">rand_arr</span><br><span class="line"><span class="comment"># array([[  5.434e-04,   2.784e-04,   4.245e-04],</span></span><br><span class="line"><span class="comment">#        [  8.448e-04,   4.719e-06,   1.216e-04],</span></span><br><span class="line"><span class="comment">#        [  6.707e-04,   8.259e-04,   1.367e-04]])</span></span><br><span class="line"><span class="comment"># 第二个操作，避免输出e-04，并指定小数点后6位数</span></span><br><span class="line">np.set_printoptions(suppress=<span class="literal">True</span>, precision=<span class="number">6</span>)</span><br><span class="line"><span class="comment"># array([[ 0.000543,  0.000278,  0.000425],</span></span><br><span class="line"><span class="comment">#        [ 0.000845,  0.000005,  0.000122],</span></span><br><span class="line"><span class="comment">#        [ 0.000671,  0.000826,  0.000137]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 含有很多元素的数组</span></span><br><span class="line">a = np.arange(<span class="number">15</span>)	<span class="comment"># 原始数组</span></span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])</span></span><br><span class="line"><span class="comment"># 设定显示元素个数的阈值</span></span><br><span class="line">np.set_printoptions(threshold=<span class="number">6</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2, ..., 12, 13, 14])</span></span><br><span class="line"><span class="comment"># 设定不隐藏元素输出</span></span><br><span class="line">np.set_printoptions(threshold=np.nan)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 导入网站数据</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">names = (<span class="string">&#x27;sepallength&#x27;</span>, <span class="string">&#x27;sepalwidth&#x27;</span>, <span class="string">&#x27;petallength&#x27;</span>, <span class="string">&#x27;petalwidth&#x27;</span>, <span class="string">&#x27;species&#x27;</span>)</span><br><span class="line">iris[:<span class="number">3</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取某一列</span></span><br><span class="line">species = np.array([row[<span class="number">4</span>] <span class="keyword">for</span> row <span class="keyword">in</span> iris])</span><br><span class="line">species</span><br><span class="line"><span class="comment"># array([b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;, b&#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;|S15&#x27;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取多列</span></span><br><span class="line"><span class="comment"># 方法一，在导入的时候指定列</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>])</span><br><span class="line"><span class="comment"># 方法二，使用上面的方法</span></span><br><span class="line">iris = np.array([row.tolist()[:<span class="number">4</span>] <span class="keyword">for</span> row <span class="keyword">in</span> iris], dtype=<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line"><span class="comment"># array([[ 5.1,  3.5,  1.4,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.9,  3. ,  1.4,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.7,  3.2,  1.3,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.6,  3.1,  1.5,  0.2]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算平均值、方差、中位数</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line">mu, med, sd = np.mean(sepallength), np.median(sepallength), np.std(sepallength)</span><br><span class="line"><span class="comment"># 5.84333333333 5.8 0.825301291785</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 0-1化数组</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line">Smax, Smin = sepallength.<span class="built_in">max</span>(), sepallength.<span class="built_in">min</span>()</span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">S = (sepallength - Smin)/(Smax - Smin)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">S = (sepallength - Smin)/sepallength.ptp()</span><br><span class="line"><span class="comment"># array([ 0.222222,  0.166667,  0.111111, ...,  0.611111,  0.527778,</span></span><br><span class="line"><span class="comment">#         0.444444])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># softmax数组</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">softmax</span>(<span class="params">x</span>):</span><br><span class="line">    e_x = np.exp(x - np.<span class="built_in">max</span>(x))</span><br><span class="line">    <span class="keyword">return</span> e_x / e_x.<span class="built_in">sum</span>(axis=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">softmax(sepallength)</span><br><span class="line"><span class="comment"># array([ 0.00222 ,  0.001817,  0.001488, ...,  0.009001,  0.006668,  0.00494 ])</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-3">31-40</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 提取数据中百分位数</span></span><br><span class="line">sepallength = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>])</span><br><span class="line"><span class="comment"># 提取数据中5%, 95%的数</span></span><br><span class="line">np.percentile(sepallength, q=[<span class="number">5</span>, <span class="number">95</span>])</span><br><span class="line"><span class="comment"># array([ 4.6  ,  7.255])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将数据中的缺失值替换为随机数</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">i, j = np.where(iris_2d)</span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">iris_2d[np.random.choice((i), <span class="number">20</span>), np.random.choice((j), <span class="number">20</span>)] = np.nan</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">iris_2d[np.random.randint(<span class="number">150</span>, size=<span class="number">20</span>), np.random.randint(<span class="number">4</span>, size=<span class="number">20</span>)] = np.nan</span><br><span class="line">iris_2d[:<span class="number">10</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.0&#x27;, b&#x27;3.6&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.4&#x27;, b&#x27;3.9&#x27;, b&#x27;1.7&#x27;, b&#x27;0.4&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.4&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;5.0&#x27;, b&#x27;3.4&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, nan, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计数据中缺失值位置</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>,<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">iris_2d[np.random.randint(<span class="number">150</span>, size=<span class="number">20</span>), np.random.randint(<span class="number">4</span>, size=<span class="number">20</span>)] = np.nan</span><br><span class="line"><span class="comment"># 统计缺失值个数</span></span><br><span class="line">np.isnan(iris_2d[:, <span class="number">0</span>]).<span class="built_in">sum</span>()</span><br><span class="line"><span class="comment"># 5</span></span><br><span class="line"><span class="comment"># 统计缺失值位置</span></span><br><span class="line">np.where(np.isnan(iris_2d[:, <span class="number">0</span>]))</span><br><span class="line"><span class="comment"># (array([ 38,  80, 106, 113, 121], dtype=int64),)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组多重条件筛选</span></span><br><span class="line">condition = (iris_2d[:, <span class="number">2</span>] &gt; <span class="number">1.5</span>) &amp; (iris_2d[:, <span class="number">0</span>] &lt; <span class="number">5.0</span>)</span><br><span class="line">iris_2d[condition]</span><br><span class="line"><span class="comment"># array([[ 4.8,  3.4,  1.6,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.8,  3.4,  1.9,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.7,  3.2,  1.6,  0.2],</span></span><br><span class="line"><span class="comment">#        [ 4.8,  3.1,  1.6,  nan],</span></span><br><span class="line"><span class="comment">#        [ 4.9,  2.4,  3.3,  1. ]])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除有缺失值的行，注意~</span></span><br><span class="line">any_nan_in_row = np.array([~np.<span class="built_in">any</span>(np.isnan(row)) <span class="keyword">for</span> row <span class="keyword">in</span> iris_2d])</span><br><span class="line"><span class="comment"># 未删除前</span></span><br><span class="line">iris_2d.shape</span><br><span class="line"><span class="comment"># (150, 4)</span></span><br><span class="line"><span class="comment"># 删除后</span></span><br><span class="line">iris_2d[any_nan_in_row].shape()</span><br><span class="line"><span class="comment"># (130, 4)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算数组间关联度</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;float&#x27;</span>, usecols=[<span class="number">0</span>,<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">np.corrcoef(iris[:, <span class="number">0</span>], iris[:, <span class="number">2</span>])[<span class="number">0</span>, <span class="number">1</span>]</span><br><span class="line"><span class="comment"># 0.87175415730487182</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 检查数组中是否含有缺失值</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># True</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 替换数组中的缺失值为0</span></span><br><span class="line">iris_2d[np.isnan(iris_2d)] = <span class="number">0</span></span><br><span class="line"><span class="comment"># 替换前</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># True</span></span><br><span class="line"><span class="comment"># 替换后</span></span><br><span class="line">np.isnan(iris_2d).<span class="built_in">any</span>()</span><br><span class="line"><span class="comment"># False</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计数组中非重复数值的情况</span></span><br><span class="line">np.unique(iris, return_counts=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># 上面的数组的表示数字，下面的数组表示出现次数</span></span><br><span class="line"><span class="comment"># (array([ 0.1,  0.2,  0.3,  0.4,  0.5,  0.6,  1. ,  1.1,  1.2,  1.3,  1.4,</span></span><br><span class="line"><span class="comment">#          1.5,  1.6,  1.7,  1.8,  1.9,  2. ,  2.1,  2.2,  2.3,  2.4,  2.5,</span></span><br><span class="line"><span class="comment">#          2.6,  2.7,  2.8,  2.9,  3. ,  3.1,  3.2,  3.3,  3.4,  3.5,  3.6,</span></span><br><span class="line"><span class="comment">#          3.7,  3.8,  3.9,  4. ,  4.1,  4.2,  4.3,  4.4,  4.5,  4.6,  4.7,</span></span><br><span class="line"><span class="comment">#          4.8,  4.9,  5. ,  5.1,  5.2,  5.3,  5.4,  5.5,  5.6,  5.7,  5.8,</span></span><br><span class="line"><span class="comment">#          5.9,  6. ,  6.1,  6.2,  6.3,  6.4,  6.5,  6.6,  6.7,  6.8,  6.9,</span></span><br><span class="line"><span class="comment">#          7. ,  7.1,  7.2,  7.3,  7.4,  7.6,  7.7,  7.9]),</span></span><br><span class="line"><span class="comment">#  array([ 6, 28,  7,  7,  1,  1,  8,  4,  7, 20, 20, 26, 11,  6, 12,  7,  7,</span></span><br><span class="line"><span class="comment">#          6,  6, 12,  6, 11,  5,  9, 14, 10, 27, 12, 13,  8, 12,  8,  4,  4,</span></span><br><span class="line"><span class="comment">#          7,  5,  6,  4,  5,  3,  8,  9,  7,  7,  9, 11, 14, 17,  6,  3,  8,</span></span><br><span class="line"><span class="comment">#         10, 12, 11, 10,  5,  8,  9,  4, 10,  8,  5,  3, 10,  3,  5,  1,  1,</span></span><br><span class="line"><span class="comment">#          3,  1,  1,  1,  4,  1], dtype=int64))</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将数组中处于相应区间的数换位文本</span></span><br><span class="line"><span class="comment"># 将数组中数组位于[0, 3, 5, 10]区间内的转换为对应的索引</span></span><br><span class="line">petal_length_bin = np.digitize(iris[:, <span class="number">2</span>].astype(<span class="string">&#x27;float&#x27;</span>), [<span class="number">0</span>, <span class="number">3</span>, <span class="number">5</span>, <span class="number">10</span>])</span><br><span class="line">label_map = &#123;<span class="number">1</span>: <span class="string">&#x27;small&#x27;</span>, <span class="number">2</span>: <span class="string">&#x27;medium&#x27;</span>, <span class="number">3</span>: <span class="string">&#x27;large&#x27;</span>, <span class="number">4</span>: np.nan&#125;</span><br><span class="line">petal_length_cat = [label_map[x] <span class="keyword">for</span> x <span class="keyword">in</span> petal_length_bin]</span><br><span class="line">petal_length_cat[:<span class="number">4</span>]</span><br><span class="line"><span class="comment"># [&#x27;small&#x27;, &#x27;small&#x27;, &#x27;small&#x27;, &#x27;small&#x27;]</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-4">41-50</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在已经存在的数组中，多加一列</span></span><br><span class="line">iris_2d = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line"><span class="comment"># 计算</span></span><br><span class="line">sepallength = iris_2d[:, <span class="number">0</span>].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">petallength = iris_2d[:, <span class="number">2</span>].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">volume = (np.pi * petallength * (sepallength**<span class="number">2</span>))/<span class="number">3</span></span><br><span class="line">volume = volume[:, np.newaxis]</span><br><span class="line"><span class="comment"># 横向堆叠</span></span><br><span class="line">out = np.hstack([iris_2d, volume])</span><br><span class="line">out[:<span class="number">4</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;5.1&#x27;, b&#x27;3.5&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 38.13265162927291],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 35.200498485922445],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 30.0723720777127],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;, 33.238050274980004]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对数组进行概率抽样</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">species = iris[:, <span class="number">4</span>]</span><br><span class="line"><span class="comment"># 方法一：生成概率抽样</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.array([<span class="string">&#x27;Iris-setosa&#x27;</span>, <span class="string">&#x27;Iris-versicolor&#x27;</span>, <span class="string">&#x27;Iris-virginica&#x27;</span>])</span><br><span class="line">species_out = np.random.choice(a, <span class="number">150</span>, p=[<span class="number">0.5</span>, <span class="number">0.25</span>, <span class="number">0.25</span>])</span><br><span class="line"><span class="comment"># 方法二：直接概率抽样</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">probs = np.r_[np.linspace(<span class="number">0</span>, <span class="number">0.500</span>, num=<span class="number">50</span>), np.linspace(<span class="number">0.501</span>, <span class="number">.750</span>, num=<span class="number">50</span>), np.linspace(<span class="number">.751</span>, <span class="number">1.0</span>, num=<span class="number">50</span>)]</span><br><span class="line">index = np.searchsorted(probs, np.random.random(<span class="number">150</span>))</span><br><span class="line">species_out = species[index]</span><br><span class="line">np.unique(species_out, return_counts=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># (array([b&#x27;Iris-setosa&#x27;, b&#x27;Iris-versicolor&#x27;, b&#x27;Iris-virginica&#x27;], dtype=object), array([77, 37, 36], dtype=int64))</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对筛选后的数组，取出第二大的数</span></span><br><span class="line">petal_len_setosa = iris[iris[:, <span class="number">4</span>] == <span class="string">b&#x27;Iris-setosa&#x27;</span>, [<span class="number">2</span>]].astype(<span class="string">&#x27;float&#x27;</span>)</span><br><span class="line">np.unique(np.sort(petal_len_setosa))[-<span class="number">2</span>]</span><br><span class="line"><span class="comment"># 1.7</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 对二维数组按照其中某一列进行排序</span></span><br><span class="line"><span class="comment"># 按照第一列升序</span></span><br><span class="line">iris[iris[:,<span class="number">0</span>].argsort()][:<span class="number">20</span>]</span><br><span class="line"><span class="comment"># array([[b&#x27;4.3&#x27;, b&#x27;3.0&#x27;, b&#x27;1.1&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;3.0&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.4&#x27;, b&#x27;2.9&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.5&#x27;, b&#x27;2.3&#x27;, b&#x27;1.3&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.6&#x27;, b&#x27;1.0&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.4&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.6&#x27;, b&#x27;3.2&#x27;, b&#x27;1.4&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.3&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.7&#x27;, b&#x27;3.2&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.0&#x27;, b&#x27;1.4&#x27;, b&#x27;0.3&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.4&#x27;, b&#x27;1.9&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.4&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.8&#x27;, b&#x27;3.1&#x27;, b&#x27;1.6&#x27;, b&#x27;0.2&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;2.4&#x27;, b&#x27;3.3&#x27;, b&#x27;1.0&#x27;, b&#x27;Iris-versicolor&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;2.5&#x27;, b&#x27;4.5&#x27;, b&#x27;1.7&#x27;, b&#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;],</span></span><br><span class="line"><span class="comment">#        [b&#x27;4.9&#x27;, b&#x27;3.1&#x27;, b&#x27;1.5&#x27;, b&#x27;0.1&#x27;, b&#x27;Iris-setosa&#x27;]], dtype=object)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计出现次数最多的数</span></span><br><span class="line">vals, counts = np.unique(iris[:, <span class="number">3</span>], return_counts=<span class="literal">True</span>)</span><br><span class="line">vals[np.argmax(counts)]</span><br><span class="line"><span class="comment"># b&#x27;0.2&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中第一次出现比给定数字大的值的缩影</span></span><br><span class="line"><span class="comment"># 这里设定要找比1.0大的值</span></span><br><span class="line">np.argmax(iris[:, <span class="number">3</span>].astype(<span class="string">&#x27;float&#x27;</span>) &gt; <span class="number">1.0</span>)</span><br><span class="line"><span class="comment"># 50</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 给定一个区间，如果数字不在这个区间内，就替换为区间的最小/最大值</span></span><br><span class="line">a = np.random.uniform(<span class="number">1</span>,<span class="number">50</span>, <span class="number">20</span>)  <span class="comment"># 原始数组</span></span><br><span class="line"><span class="comment"># array([ 22.153505  ,  47.06146116,  41.06481956,  17.46948556,</span></span><br><span class="line"><span class="comment">#          9.59511223,  19.26877027,   1.27873686,  13.36889132,</span></span><br><span class="line"><span class="comment">#         39.98746292,   1.74749359,  30.34332547,  30.58642241,</span></span><br><span class="line"><span class="comment">#          6.15223659,  19.7152288 ,   2.78732677,  44.63016661,</span></span><br><span class="line"><span class="comment">#         49.06512199,   3.93715745,  44.63675129,  29.26817347])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">np.clip(a, a_min=<span class="number">10</span>, a_max=<span class="number">30</span>)</span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.where(a &lt; <span class="number">10</span>, <span class="number">10</span>, np.where(a &gt; <span class="number">30</span>, <span class="number">30</span>, a))</span><br><span class="line"><span class="comment"># array([ 22.153505  ,  30.        ,  30.        ,  17.46948556,</span></span><br><span class="line"><span class="comment">#         10.        ,  19.26877027,  10.        ,  13.36889132,</span></span><br><span class="line"><span class="comment">#         30.        ,  10.        ,  30.        ,  30.        ,</span></span><br><span class="line"><span class="comment">#         10.        ,  19.7152288 ,  10.        ,  30.        ,</span></span><br><span class="line"><span class="comment">#         30.        ,  10.        ,  30.        ,  29.26817347])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 统计前n大的元素位置</span></span><br><span class="line">a = np.random.uniform(<span class="number">1</span>,<span class="number">50</span>, <span class="number">20</span>)  <span class="comment"># 原始数组</span></span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([ 22.153505  ,  47.06146116,  41.06481956,  17.46948556,</span></span><br><span class="line"><span class="comment">#          9.59511223,  19.26877027,   1.27873686,  13.36889132,</span></span><br><span class="line"><span class="comment">#         39.98746292,   1.74749359,  30.34332547,  30.58642241,</span></span><br><span class="line"><span class="comment">#          6.15223659,  19.7152288 ,   2.78732677,  44.63016661,</span></span><br><span class="line"><span class="comment">#         49.06512199,   3.93715745,  44.63675129,  29.26817347])</span></span><br><span class="line"><span class="comment"># 方法一：</span></span><br><span class="line">a.argsort()[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># array([ 2, 15, 18,  1, 16], dtype=int64)</span></span><br><span class="line"><span class="comment"># 方法二：</span></span><br><span class="line">np.argpartition(-a, <span class="number">5</span>)[:<span class="number">5</span>]</span><br><span class="line"><span class="comment"># array([16,  1, 18, 15,  2], dtype=int64)</span></span><br><span class="line"><span class="comment"># 获取前n大的元素</span></span><br><span class="line">a[a.argsort()][-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法1:</span></span><br><span class="line">a[a.argsort()][-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法2:</span></span><br><span class="line">np.sort(a)[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 方法3:</span></span><br><span class="line">np.partition(a, kth=-<span class="number">5</span>)[-<span class="number">5</span>:]</span><br><span class="line"><span class="comment"># 前三个输出都是</span></span><br><span class="line"><span class="comment"># array([ 41.06481956,  44.63016661,  47.06146116,  44.63675129,  49.06512199])</span></span><br><span class="line"><span class="comment"># 方法4:</span></span><br><span class="line">a[np.argpartition(-a, <span class="number">5</span>)][:<span class="number">5</span>]</span><br><span class="line"><span class="comment"># array([ 49.06512199,  47.06146116,  44.63675129,  44.63016661,  41.06481956])</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算有唯一值的行数,没看懂</span></span><br><span class="line">arr = np.random.randint(<span class="number">1</span>,<span class="number">11</span>,size=(<span class="number">6</span>, <span class="number">10</span>))</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([[ 2,  6,  5,  3,  9,  4,  6,  1, 10,  4],</span></span><br><span class="line"><span class="comment">#        [ 7,  4,  5,  8,  7,  4, 10,  1,  5,  5],</span></span><br><span class="line"><span class="comment">#        [ 6,  8,  7,  7,  3,  5,  3,  8,  2,  7],</span></span><br><span class="line"><span class="comment">#        [ 7,  1,  8,  3,  4,  6,  5,  3,  5,  4],</span></span><br><span class="line"><span class="comment">#        [ 8, 10,  1,  1,  6, 10,  7,  7,  6,  7],</span></span><br><span class="line"><span class="comment">#        [ 5,  8,  4, 10,  3,  4,  9,  8,  2,  6]])</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">counts_of_all_values_rowwise</span>(<span class="params">arr2d</span>):</span><br><span class="line">    num_counts_array = [np.unique(row, return_counts=<span class="literal">True</span>) <span class="keyword">for</span> row <span class="keyword">in</span> arr2d]</span><br><span class="line">    <span class="keyword">return</span> [[<span class="built_in">int</span>(b[a==i]) <span class="keyword">if</span> i <span class="keyword">in</span> a <span class="keyword">else</span> <span class="number">0</span> <span class="keyword">for</span> i <span class="keyword">in</span> np.unique(arr2d)] <span class="keyword">for</span> a, b <span class="keyword">in</span> num_counts_array]</span><br><span class="line">counts_of_all_values_rowwise(arr)</span><br><span class="line"><span class="comment"># [[1, 1, 1, 2, 1, 2, 0, 0, 1, 1],</span></span><br><span class="line"><span class="comment">#  [1, 0, 0, 2, 3, 0, 2, 1, 0, 1],</span></span><br><span class="line"><span class="comment">#  [0, 1, 2, 0, 1, 1, 3, 2, 0, 0],</span></span><br><span class="line"><span class="comment">#  [1, 0, 2, 2, 2, 1, 1, 1, 0, 0],</span></span><br><span class="line"><span class="comment">#  [2, 0, 0, 0, 0, 2, 3, 1, 0, 2],</span></span><br><span class="line"><span class="comment">#  [0, 1, 1, 2, 1, 1, 0, 2, 1, 1]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将多维数组转换为一维数组</span></span><br><span class="line">arr1 = np.arange(<span class="number">3</span>)</span><br><span class="line">arr2 = np.arange(<span class="number">3</span>,<span class="number">7</span>)</span><br><span class="line">arr3 = np.arange(<span class="number">7</span>,<span class="number">10</span>)</span><br><span class="line">array_of_arrays = np.array([arr1, arr2, arr3])</span><br><span class="line">array_of_arrays      <span class="comment"># 原始数组</span></span><br><span class="line"><span class="comment"># array([array([0, 1, 2]), array([3, 4, 5, 6]), array([7, 8, 9])], dtype=object)</span></span><br><span class="line">arr_2d = np.concatenate(array_of_arrays)</span><br><span class="line">arr_2d</span><br><span class="line"><span class="comment"># array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])</span></span><br></pre></td></tr></table></figure>
+<h2 id="section-5">51-60</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 唯一编码位置</span></span><br><span class="line">np.random.seed(<span class="number">101</span>)</span><br><span class="line">arr = np.random.randint(<span class="number">1</span>,<span class="number">4</span>, size=<span class="number">6</span>)</span><br><span class="line">arr</span><br><span class="line"><span class="comment"># array([2, 3, 2, 2, 2, 1])</span></span><br><span class="line"><span class="comment"># 返回数组是3x6,因为数组长度为6，最大数为1。1对应的是[1,0,0,],2对应的是[0,1,0],3对应的是[0,0,1]</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">one_hot_encodings</span>(<span class="params">arr</span>):</span><br><span class="line">    uniqs = np.unique(arr)</span><br><span class="line">    out = np.zeros((arr.shape[<span class="number">0</span>], uniqs.shape[<span class="number">0</span>]))</span><br><span class="line">    <span class="keyword">for</span> i, k <span class="keyword">in</span> <span class="built_in">enumerate</span>(arr):</span><br><span class="line">        out[i, k-<span class="number">1</span>] = <span class="number">1</span></span><br><span class="line">    <span class="keyword">return</span> out</span><br><span class="line">one_hot_encodings(arr)</span><br><span class="line"><span class="comment"># array([[ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  0.,  1.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 0.,  1.,  0.],</span></span><br><span class="line"><span class="comment">#        [ 1.,  0.,  0.]])</span></span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">(arr[:, <span class="literal">None</span>] == np.unique(arr)).view(np.int8)</span><br><span class="line"><span class="comment"># array([[0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 0, 1],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [0, 1, 0],</span></span><br><span class="line"><span class="comment">#        [1, 0, 0]], dtype=int8)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取由变量名分组的行号</span></span><br><span class="line">url = <span class="string">&#x27;https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data&#x27;</span></span><br><span class="line">species = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;str&#x27;</span>, usecols=<span class="number">4</span>)</span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">species_small = np.sort(np.random.choice(species, size=<span class="number">20</span>))</span><br><span class="line">species_small</span><br><span class="line"><span class="comment"># array([&#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-setosa&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;&lt;U15&#x27;)</span></span><br><span class="line"></span><br><span class="line">[i <span class="keyword">for</span> val <span class="keyword">in</span> np.unique(species_small) <span class="keyword">for</span> i, grp <span class="keyword">in</span> <span class="built_in">enumerate</span>(species_small[species_small==val])]</span><br><span class="line">[<span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>, <span class="number">7</span>, <span class="number">8</span>, <span class="number">0</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">5</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据已知分类组名创建索引</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">species_small = np.sort(np.random.choice(species, size=<span class="number">20</span>))</span><br><span class="line">species_small</span><br><span class="line"><span class="comment"># array([&#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-setosa&#x27;, &#x27;Iris-setosa&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;, &#x27;Iris-versicolor&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-versicolor&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;, &#x27;Iris-virginica&#x27;],</span></span><br><span class="line"><span class="comment">#       dtype=&#x27;&lt;U15&#x27;)</span></span><br><span class="line"><span class="comment"># 方法一，用列表解析式</span></span><br><span class="line">output = [np.argwhere(np.unique(species_small) == s).tolist()[<span class="number">0</span>][<span class="number">0</span>] <span class="keyword">for</span> val <span class="keyword">in</span> np.unique(species_small) <span class="keyword">for</span> s <span class="keyword">in</span> species_small[species_small==val]]</span><br><span class="line"><span class="comment"># 方法二，不用列表解析式直接循环</span></span><br><span class="line">output = []</span><br><span class="line">uniqs = np.unique(species_small)</span><br><span class="line"><span class="keyword">for</span> val <span class="keyword">in</span> uniqs:  <span class="comment"># uniq values in group</span></span><br><span class="line">    <span class="keyword">for</span> s <span class="keyword">in</span> species_small[species_small==val]:  <span class="comment"># each element in group</span></span><br><span class="line">        groupid = np.argwhere(uniqs == s).tolist()[<span class="number">0</span>][<span class="number">0</span>]  <span class="comment"># groupid</span></span><br><span class="line">        output.append(groupid)</span><br><span class="line">output</span><br><span class="line"><span class="comment"># [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数值排序，输出对应的索引值</span></span><br><span class="line">np.random.seed(<span class="number">10</span>)</span><br><span class="line">a = np.random.randint(<span class="number">20</span>, size=<span class="number">10</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [ 9  4 15  0 17 16 17  8  9  0]</span></span><br><span class="line">a.argsort().argsort()</span><br><span class="line"><span class="comment"># [4 2 6 0 8 7 9 3 5 1]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 多维数组排序，输出对应的索引值</span></span><br><span class="line">np.random.seed(<span class="number">10</span>)</span><br><span class="line">a = np.random.randint(<span class="number">20</span>, size=[<span class="number">2</span>,<span class="number">5</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [[ 9  4 15  0 17]</span></span><br><span class="line"><span class="comment">#  [16 17  8  9  0]]</span></span><br><span class="line">a.ravel().argsort().argsort().reshape(a.shape)</span><br><span class="line"><span class="comment"># [[ 9  4 15  0 17]</span></span><br><span class="line"><span class="comment">#  [16 17  8  9  0]]</span></span><br><span class="line"><span class="comment"># [[4 2 6 0 8]</span></span><br><span class="line"><span class="comment">#  [7 9 3 5 1]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每一行的最大值</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">1</span>,<span class="number">10</span>, [<span class="number">5</span>,<span class="number">3</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([[9, 9, 4],</span></span><br><span class="line"><span class="comment">#        [8, 8, 1],</span></span><br><span class="line"><span class="comment">#        [5, 3, 6],</span></span><br><span class="line"><span class="comment">#        [3, 3, 3],</span></span><br><span class="line"><span class="comment">#        [2, 1, 9]])</span></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line">np.amax(a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">np.apply_along_axis(np.<span class="built_in">max</span>, arr=a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># array([9, 8, 6, 3, 9])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每行最小值比最大值</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">1</span>,<span class="number">10</span>, [<span class="number">5</span>,<span class="number">3</span>])</span><br><span class="line">a</span><br><span class="line"><span class="comment"># array([[9, 9, 4],</span></span><br><span class="line"><span class="comment">#        [8, 8, 1],</span></span><br><span class="line"><span class="comment">#        [5, 3, 6],</span></span><br><span class="line"><span class="comment">#        [3, 3, 3],</span></span><br><span class="line"><span class="comment">#        [2, 1, 9]])</span></span><br><span class="line">np.apply_along_axis(<span class="keyword">lambda</span> x: np.<span class="built_in">min</span>(x)/np.<span class="built_in">max</span>(x), arr=a, axis=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># array([ 0.44444444,  0.125     ,  0.5       ,  1.        ,  0.11111111])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中重复的数值，输出布尔型</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">a = np.random.randint(<span class="number">0</span>, <span class="number">5</span>, <span class="number">10</span>)</span><br><span class="line">a</span><br><span class="line"><span class="comment"># [0 0 3 0 2 4 2 2 2 2]</span></span><br><span class="line">out = np.full(a.shape[<span class="number">0</span>], <span class="literal">True</span>)</span><br><span class="line">unique_positions = np.unique(a, return_index=<span class="literal">True</span>)[<span class="number">1</span>]</span><br><span class="line">out[unique_positions] = <span class="literal">False</span></span><br><span class="line">out</span><br><span class="line"><span class="comment"># [False  True False  True False False  True  True  True  True]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出多维数组每一分类的平均值</span></span><br><span class="line">iris = np.genfromtxt(url, delimiter=<span class="string">&#x27;,&#x27;</span>, dtype=<span class="string">&#x27;object&#x27;</span>)</span><br><span class="line">names = (<span class="string">&#x27;sepallength&#x27;</span>, <span class="string">&#x27;sepalwidth&#x27;</span>, <span class="string">&#x27;petallength&#x27;</span>, <span class="string">&#x27;petalwidth&#x27;</span>, <span class="string">&#x27;species&#x27;</span>)</span><br><span class="line">numeric_column = iris[:, <span class="number">1</span>].astype(<span class="string">&#x27;float&#x27;</span>)  <span class="comment"># sepalwidth</span></span><br><span class="line">grouping_column = iris[:, <span class="number">4</span>]  <span class="comment"># species</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法一：列表解析</span></span><br><span class="line">output = [[group_val, numeric_column[grouping_column==group_val].mean()] <span class="keyword">for</span> group_val <span class="keyword">in</span> np.unique(grouping_column)]</span><br><span class="line"><span class="comment"># 方法二：不用列表解析直接循环</span></span><br><span class="line">output = []</span><br><span class="line"><span class="keyword">for</span> group_val <span class="keyword">in</span> np.unique(grouping_column):</span><br><span class="line">    output.append([group_val, numeric_column[grouping_column==group_val].mean()])</span><br><span class="line">output</span><br><span class="line"><span class="comment"># [[b&#x27;Iris-setosa&#x27;, 3.418],</span></span><br><span class="line"><span class="comment">#  [b&#x27;Iris-versicolor&#x27;, 2.770],</span></span><br><span class="line"><span class="comment">#  [b&#x27;Iris-virginica&#x27;, 2.974]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将图像转换为数组存储，并再转换出图像显示</span></span><br><span class="line"><span class="keyword">from</span> io <span class="keyword">import</span> BytesIO</span><br><span class="line"><span class="keyword">from</span> PIL <span class="keyword">import</span> Image</span><br><span class="line"><span class="keyword">import</span> PIL, requests</span><br><span class="line"></span><br><span class="line">URL = <span class="string">&#x27;https://upload.wikimedia.org/wikipedia/commons/8/8b/Denali_Mt_McKinley.jpg&#x27;</span></span><br><span class="line">response = requests.get(URL)</span><br><span class="line">I = Image.<span class="built_in">open</span>(BytesIO(response.content))</span><br><span class="line">I = I.resize([<span class="number">150</span>,<span class="number">150</span>])</span><br><span class="line">arr = np.asarray(I)</span><br><span class="line">im = PIL.Image.fromarray(np.uint8(arr))</span><br><span class="line">Image.Image.show(im)</span><br></pre></td></tr></table></figure>
+<h2 id="section-6">61-70</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line"><span class="comment"># 去除数组中所有的缺失值</span></span><br><span class="line">a = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,np.nan,<span class="number">5</span>,<span class="number">6</span>,<span class="number">7</span>,np.nan])</span><br><span class="line">a[~np.isnan(a)]</span><br><span class="line">array([ <span class="number">1.</span>,  <span class="number">2.</span>,  <span class="number">3.</span>,  <span class="number">5.</span>,  <span class="number">6.</span>,  <span class="number">7.</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 计算两个数组间的欧几里得</span></span><br><span class="line">a = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>,<span class="number">4</span>,<span class="number">5</span>])</span><br><span class="line">b = np.array([<span class="number">4</span>,<span class="number">5</span>,<span class="number">6</span>,<span class="number">7</span>,<span class="number">8</span>])</span><br><span class="line">dist = np.linalg.norm(a-b)</span><br><span class="line">dist</span><br><span class="line"><span class="comment"># 6.7082039324993694</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找出数组中最大的两个数</span></span><br><span class="line">a = np.array([<span class="number">1</span>, <span class="number">3</span>, <span class="number">7</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">6</span>, <span class="number">0</span>, <span class="number">1</span>])</span><br><span class="line">doublediff = np.diff(np.sign(np.diff(a)))</span><br><span class="line">peak_locations = np.where(doublediff == -<span class="number">2</span>)[<span class="number">0</span>] + <span class="number">1</span></span><br><span class="line">peak_locations</span><br><span class="line"><span class="comment"># array([2, 5])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 二维数组数组数值每项减去一维数组对应的行号</span></span><br><span class="line">a_2d = np.array([[<span class="number">3</span>,<span class="number">3</span>,<span class="number">3</span>],[<span class="number">4</span>,<span class="number">4</span>,<span class="number">4</span>],[<span class="number">5</span>,<span class="number">5</span>,<span class="number">5</span>]])</span><br><span class="line">b_1d = np.array([<span class="number">1</span>,<span class="number">2</span>,<span class="number">3</span>])</span><br><span class="line">a_2d - b_1d[:,<span class="literal">None</span>]</span><br><span class="line"><span class="comment"># [[2 2 2]</span></span><br><span class="line"><span class="comment">#  [2 2 2]</span></span><br><span class="line"><span class="comment">#  [2 2 2]]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 数组中重复的第n次的数的索引</span></span><br><span class="line">x = np.array([<span class="number">1</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">3</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>])</span><br><span class="line">n = <span class="number">5</span></span><br><span class="line"><span class="comment"># 方法一：列表解析</span></span><br><span class="line">[i <span class="keyword">for</span> i, v <span class="keyword">in</span> <span class="built_in">enumerate</span>(x) <span class="keyword">if</span> v == <span class="number">1</span>][n-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># 方法二：numpy方法</span></span><br><span class="line">np.where(x == <span class="number">1</span>)[<span class="number">0</span>][n-<span class="number">1</span>]</span><br><span class="line"><span class="comment"># 8</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 将`datetime64`格式转换为`datetime`</span></span><br><span class="line">dt64 = np.datetime64(<span class="string">&#x27;2018-02-25 22:10:10&#x27;</span>)</span><br><span class="line"><span class="comment"># 方法一：利用datetime模块</span></span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime</span><br><span class="line">dt64.tolist()</span><br><span class="line"><span class="comment"># 方法二：numpy直接转换</span></span><br><span class="line">dt64.astype(datetime)</span><br><span class="line">datetime.datetime(<span class="number">2018</span>, <span class="number">2</span>, <span class="number">25</span>, <span class="number">22</span>, <span class="number">10</span>, <span class="number">10</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 这个没看懂。。</span></span><br><span class="line"><span class="comment"># 如何计算numpy数组的移动平均值(累加和)</span></span><br><span class="line">np.random.seed(<span class="number">100</span>)</span><br><span class="line">Z = np.random.randint(<span class="number">10</span>, size=<span class="number">10</span>)</span><br><span class="line">Z</span><br><span class="line"><span class="comment"># [8 8 3 7 7 0 4 2 5 2]</span></span><br><span class="line"><span class="comment"># 方法一: np.cumsum</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">moving_average</span>(<span class="params">a, n=<span class="number">3</span></span>) :</span><br><span class="line">    ret = np.cumsum(a, dtype=<span class="built_in">float</span>)</span><br><span class="line">    ret[n:] = ret[n:] - ret[:-n]</span><br><span class="line">    <span class="keyword">return</span> ret[n - <span class="number">1</span>:] / n</span><br><span class="line">moving_average(Z, n=<span class="number">3</span>).<span class="built_in">round</span>(<span class="number">2</span>)</span><br><span class="line"><span class="comment"># 方法二: np.convolve</span></span><br><span class="line">np.convolve(Z, np.ones(<span class="number">3</span>)/<span class="number">3</span>, mode=<span class="string">&#x27;valid&#x27;</span>)</span><br><span class="line"><span class="comment"># [ 6.33  6.    5.67  4.67  3.67  2.    3.67  3.  ]</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 创建一个数组，只给出起始值、步长、数组长度</span></span><br><span class="line">length = <span class="number">10</span></span><br><span class="line">start = <span class="number">5</span></span><br><span class="line">step = <span class="number">3</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">seq</span>(<span class="params">start, length, step</span>):</span><br><span class="line">    end = start + (step*length)</span><br><span class="line">    <span class="keyword">return</span> np.arange(start, end, step)</span><br><span class="line">seq(start, length, step)</span><br><span class="line"><span class="comment"># array([ 5,  8, 11, 14, 17, 20, 23, 26, 29, 32])</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># numpy数组填补不完整的日期</span></span><br><span class="line">dates = np.arange(np.datetime64(<span class="string">&#x27;2018-02-01&#x27;</span>), np.datetime64(<span class="string">&#x27;2018-02-25&#x27;</span>), <span class="number">2</span>)</span><br><span class="line">dates</span><br><span class="line"><span class="comment"># [&#x27;2018-02-01&#x27; &#x27;2018-02-03&#x27; &#x27;2018-02-05&#x27; &#x27;2018-02-07&#x27; &#x27;2018-02-09&#x27;</span></span><br><span class="line"><span class="comment">#  &#x27;2018-02-11&#x27; &#x27;2018-02-13&#x27; &#x27;2018-02-15&#x27; &#x27;2018-02-17&#x27; &#x27;2018-02-19&#x27;</span></span><br><span class="line"><span class="comment">#  &#x27;2018-02-21&#x27; &#x27;2018-02-23&#x27;]</span></span><br><span class="line"><span class="comment"># 方法一: 列表解析</span></span><br><span class="line">filled_in = np.array([np.arange(date, (date+d)) <span class="keyword">for</span> date, d <span class="keyword">in</span> <span class="built_in">zip</span>(dates, np.diff(dates))]).reshape(-<span class="number">1</span>)</span><br><span class="line">output = np.hstack([filled_in, dates[-<span class="number">1</span>]])</span><br><span class="line"><span class="comment"># 方法二: 不用列表解析循环</span></span><br><span class="line">out = []</span><br><span class="line"><span class="keyword">for</span> date, d <span class="keyword">in</span> <span class="built_in">zip</span>(dates, np.diff(dates)):</span><br><span class="line">    out.append(np.arange(date, (date+d)))</span><br><span class="line">filled_in = np.array(out).reshape(-<span class="number">1</span>)</span><br><span class="line">output = np.hstack([filled_in, dates[-<span class="number">1</span>]])</span><br><span class="line">output</span><br><span class="line"><span class="comment"># array([&#x27;2018-02-01&#x27;, &#x27;2018-02-02&#x27;, &#x27;2018-02-03&#x27;, &#x27;2018-02-04&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-05&#x27;, &#x27;2018-02-06&#x27;, &#x27;2018-02-07&#x27;, &#x27;2018-02-08&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-09&#x27;, &#x27;2018-02-10&#x27;, &#x27;2018-02-11&#x27;, &#x27;2018-02-12&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-13&#x27;, &#x27;2018-02-14&#x27;, &#x27;2018-02-15&#x27;, &#x27;2018-02-16&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-17&#x27;, &#x27;2018-02-18&#x27;, &#x27;2018-02-19&#x27;, &#x27;2018-02-20&#x27;,</span></span><br><span class="line"><span class="comment">#        &#x27;2018-02-21&#x27;, &#x27;2018-02-22&#x27;, &#x27;2018-02-23&#x27;], dtype=&#x27;datetime64[D]&#x27;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 给定一维数组，变形为二维数组（给定每维长度）</span></span><br><span class="line">arr = np.arange(<span class="number">15</span>)</span><br><span class="line">arr</span><br><span class="line">array([ <span class="number">0</span>,  <span class="number">1</span>,  <span class="number">2</span>,  <span class="number">3</span>,  <span class="number">4</span>,  <span class="number">5</span>,  <span class="number">6</span>,  <span class="number">7</span>,  <span class="number">8</span>,  <span class="number">9</span>, <span class="number">10</span>, <span class="number">11</span>, <span class="number">12</span>, <span class="number">13</span>, <span class="number">14</span>])</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gen_strides</span>(<span class="params">arr, stride_len=<span class="number">5</span>, window_len=<span class="number">5</span></span>):</span><br><span class="line">    n_strides = (a.size-window_len)//stride_len + <span class="number">1</span></span><br><span class="line">    <span class="keyword">return</span> np.array([a[s:(s+window_len)] <span class="keyword">for</span> s <span class="keyword">in</span> np.arange(<span class="number">0</span>, n_strides*stride_len, stride_len)])</span><br><span class="line">gen_strides(np.arange(<span class="number">15</span>), stride_len=<span class="number">2</span>, window_len=<span class="number">4</span>)</span><br><span class="line"><span class="comment"># [[ 0  1  2  3]</span></span><br><span class="line"><span class="comment">#  [ 2  3  4  5]</span></span><br><span class="line"><span class="comment">#  [ 4  5  6  7]</span></span><br><span class="line"><span class="comment">#  [ 6  7  8  9]</span></span><br><span class="line"><span class="comment">#  [ 8  9 10 11]</span></span><br><span class="line"><span class="comment">#  [10 11 12 13]]</span></span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>Numpy</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>用 Cpp 写 PyTorch 的插件</title>
+    <url>/hexo_blog/2023/12/03/PyTorch/pytorch_cpp_extension/</url>
+    <content><![CDATA[<p>从零开始，用 Cpp 写 PyTorch 的插件，包括 CPU 和 GPU 的版本。</p>
+<p><a
+href="https://github.com/wnma3mz/pytorch_cuda_extension">代码</a></p>
+<span id="more"></span>
+<h2 id="为什么">为什么</h2>
+<p>一般来说，在原生功能不能满足需求的时候，插件可以作为补充。比如，PyTorch
+的 <code>torch.nn.functional</code> 中没有 <code>softmax</code>
+函数，但是 <code>torch.nn</code> 中有，所以可以用
+<code>torch.nn.functional.softmax</code> 来代替。但是，如果要用
+<code>softmax</code> 的导数，就需要用到 <code>softmax</code>
+的原始定义，这个时候就需要自己写插件了。</p>
+<p>如果是比较简单的需求，则可以直接用 Python
+完成。然而，当对性能要求较高时，往往会使用 Cpp
+来写插件，最后甚至会优化为 CUDA 代码。</p>
+<h2 id="怎么写">怎么写</h2>
+<p>从例子出发，一步步来写。假设要实现一个最简单的 Attention 模块，输入为
+<span class="math inline">\(q,k,v \in \mathbb{R}^{M \times
+N}\)</span>，输出为 <span class="math inline">\(out \in \mathbb{R}^{M
+\times N}\)</span>（不考虑 Batch Size 以及 Head 数量的情况）。Attention
+模块的计算公式为：</p>
+<p><span class="math display">\[
+out = \text{softmax}(qk^T)v
+\]</span></p>
+<p>只实现 <code>forward</code> 函数，不实现 <code>backward</code>
+函数。</p>
+<h3 id="cpu-版本">CPU 版本</h3>
+<p>类似于写 Python 的库，创建一个文件夹，目录结构如下所示：</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">attention</span><br><span class="line">├── attention.cpp</span><br><span class="line">├── setup.py</span><br><span class="line">└── __init__.py</span><br></pre></td></tr></table></figure>
+<h4 id="python-部分">Python 部分</h4>
+<p>核心代码在 <code>attention.cpp</code> 中 ，首先在
+<code>setup.py</code> 中添加如下代码：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> setuptools <span class="keyword">import</span> setup</span><br><span class="line"><span class="keyword">from</span> torch.utils.cpp_extension <span class="keyword">import</span> BuildExtension, CppExtension</span><br><span class="line"></span><br><span class="line">setup(</span><br><span class="line">    name=<span class="string">&#x27;attention&#x27;</span>,</span><br><span class="line">    ext_modules=[</span><br><span class="line">        CppExtension(<span class="string">&#x27;attention&#x27;</span>, [<span class="string">&#x27;attention.cpp&#x27;</span>]),</span><br><span class="line">    ],</span><br><span class="line">    cmdclass=&#123;</span><br><span class="line">        <span class="string">&#x27;build_ext&#x27;</span>: BuildExtension</span><br><span class="line">    &#125;)</span><br></pre></td></tr></table></figure>
+<p>这样，就可以用 <code>python setup.py install</code>
+来安装插件了。或者用 <code>pip install -e attention</code>
+以便于快速调试</p>
+<p>在 <code>__init__.py</code> 中导入 <code>attention</code> 模块，以在
+Python 中调用 <code>forward</code> 函数，直接计算 attention。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> .attention <span class="keyword">import</span> forward</span><br></pre></td></tr></table></figure>
+<h4 id="cpp-部分">Cpp 部分</h4>
+<p>接下来，我们需要在 <code>attention.cpp</code> 中实现
+<code>forward</code> 函数，为进行区分，这里使用这个函数名称
+<code>attention_forward</code> ，这个函数的输入是 q、k、v 三个
+<code>Tensor</code>，输出是
+<code>torch::Tensor</code>。而具体的计算步骤可以拆解为三个步骤：</p>
 <ol type="1">
-<li><p>选择最有效率的表名顺序（只在基于规则的优化器中有效）</p>
-<p><code>ORACLE</code>
-的解析器按照从右到左的顺序处理FROM子句中的表名，FROM子句中写在最后的表(基础表
-driving
-table)将被最先处理，在FROM子句中包含多个表的情况下,你必须选择记录条数最少的表作为基础表。如果有3个以上的表连接查询,
-那就需要选择交叉表(intersection table)作为基础表,
-交叉表是指那个被其他表所引用的表</p></li>
-<li><p><code>WHERE</code>子句中的连接顺序</p>
-<p><code>ORACLE</code>采用自下而上的顺序解析WHERE子句,根据这个原理,表之间的连接必须写在其他<code>WHERE</code>条件之前,
-那些可以过滤掉最大数量记录的条件必须写在WHERE子句的末尾.</p></li>
-<li><p><code>SELECT</code>子句中避免使用 '*'</p>
-<p><code>ORACLE</code>在解析的过程中, 会将'*'依次转换成所有的列名,
-这个工作是通过查询数据字典完成的, 这意味着将耗费更多的时间。</p></li>
-<li><p>减少访问数据库的次数</p>
-<p><code>ORACLE</code>在内部执行了许多工作: 解析SQL语句,
-估算索引的利用率, 绑定变量 , 读数据块等。</p></li>
-<li><p>在<code>SQL*Plus</code> ,
-<code>SQL*Forms</code>和<code>Pro*C</code>中重新设置<code>ARRAYSIZE</code>参数,
-可以增加每次数据库访问的检索数据量 ,建议值为200</p></li>
-<li><p>使用<code>DECODE</code>函数来减少处理时间</p>
-<p>使用<code>DECODE</code>函数可以避免重复扫描相同记录或重复连接相同的表.</p></li>
-<li><p>整合简单,无关联的数据库访问</p></li>
-</ol>
-<p>如果你有几个简单的数据库查询语句,你可以把它们整合到一个查询中(即使它们之间没有关系)
-。</p>
-<ol start="8" type="1">
-<li>删除重复记录：</li>
-</ol>
-<blockquote>
-<p>最高效的删除重复记录方法(因为使用了<code>ROWID</code>)</p>
-</blockquote>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">DELETE</span> <span class="keyword">FROM</span> EMP E <span class="keyword">WHERE</span> E.ROWID <span class="operator">&gt;</span> (<span class="keyword">SELECT</span> <span class="built_in">MIN</span>(X.ROWID) <span class="keyword">FROM</span> EMP X <span class="keyword">WHERE</span> X.EMP_NO <span class="operator">=</span> E.EMP_NO)</span><br></pre></td></tr></table></figure>
-<ol start="9" type="1">
-<li><p>用<code>TRUNCATE</code>替代<code>DELETE</code></p>
-<p>当删除表中的记录时,在通常情况下, 回滚段(rollback segments )
-用来存放可以被恢复的信息.
-如果你没有<code>COMMIT</code>事务,<code>ORACLE</code>会将数据恢复到删除之前的状态(准确地说是恢复到执行删除命令之前的状况)
-而当运用<code>TRUNCATE</code>时,
-回滚段不再存放任何可被恢复的信息.当命令运行后,数据不能被恢复.因此很少的资源被调用,执行时间也会很短.
-(译者按:
-<code>TRUNCATE</code>只在删除全表适用,<code>TRUNCATE</code>是DDL不是DML)
-。</p></li>
-<li><p>尽量多使用<code>COMMIT</code></p>
-<p>只要有可能,在程序中尽量多使用<code>COMMIT</code>,
-这样程序的性能得到提高,需求也会因为<code>COMMIT</code>所释放的资源而减少，<code>COMMIT</code>所释放的资源:</p>
-<ol type="a">
-<li><p>回滚段上用于恢复数据的信息.</p></li>
-<li><p>被程序语句获得的锁</p></li>
-<li><p>redo log buffer 中的空间</p></li>
-<li><p><code>ORACLE</code>为管理上述3种资源中的内部花费</p></li>
-</ol></li>
-<li><p>用<code>Where</code>子句替换<code>HAVING</code>子句</p></li>
-</ol>
-<pre><code>避免使用`HAVING`子句, `HAVING` 只会在检索出所有记录之后才对结果集进行过滤. 这个处理需要排序,总计等操作. 如果能通过`WHERE`子句限制记录的数目,那就能减少这方面的开销. (非`oracle`中)`on`、`where`、`having`这三个都可以加条件的子句中，`on`是最先执行，`where`次之，`having`最后，因为`on`是先把不符合条件的记录过滤后才进行统计，它就可以减少中间运算要处理的数据，按理说应该速度是最快的，`where`也应该比`having`快点的，因为它过滤数据后才进行`sum`，在两个表联接时才用`on`的，所以在一个表的时候，就剩下`where`跟`having`比较了。在这单表查询统计的情况下，如果要过滤的条件没有涉及到要计算字段，那它们的结果是一样的，只是`where`可以使用`rushmore`技术，而`having`就不能，在速度上后者要慢如果要涉及到计算的字 段，就表示在没计算之前，这个字段的值是不确定的，根据上篇写的工作流程，`where`的作用时间是在计算之前就完成的，而`having`就是在计算后才起作 用的，所以在这种情况下，两者的结果会不同。在多表联接查询时，`on`比`where`更早起作用。系统首先根据各个表之间的联接条件，把多个表合成一个临时表 后，再由`where`进行过滤，然后再计算，计算完后再由`having`进行过滤。由此可见，要想过滤条件起到正确的作用，首先要明白这个条件应该在什么时候起作用，然后再决定放在那里。</code></pre>
-<ol start="12" type="1">
-<li>减少对表的查询</li>
+<li>矩阵的乘法</li>
+<li>softmax</li>
+<li>矩阵的乘法</li>
 </ol>
+<p>使用 <code>PYBIND11_MODULE</code>把
+<code>attention_forward</code>函数暴露出去，绑定到
+<code>forward</code>上，这样就能用 <code>forward</code>函数来调用
+<code>attention_forward</code>。整合后的完整代码如下，</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// 参数：queries(Q)，keys(K)，values(V)</span></span><br><span class="line"><span class="comment">// 返回：方便起见，返回一个 vector，实际上只有一个元素，便于后续扩展</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor&amp; v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = torch::<span class="built_in">matmul</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>));</span><br><span class="line">    scores = torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>);</span><br><span class="line">    torch::Tensor attention = torch::<span class="built_in">matmul</span>(scores, v);</span><br><span class="line">    <span class="keyword">return</span> &#123;attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="built_in">PYBIND11_MODULE</span>(TORCH_EXTENSION_NAME, m) &#123;</span><br><span class="line">    m.<span class="built_in">def</span>(<span class="string">&quot;forward&quot;</span>, &amp;attention_forward, <span class="string">&quot;Attention forward (CPU)&quot;</span>);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h4 id="测试">测试</h4>
+<p>用 Python 版本的实现来测试 Cpp 版本的实现是否正确，测试代码如下，</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> attention</span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line">torch.manual_seed(<span class="number">42</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">py_attention</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    <span class="keyword">return</span> torch.softmax(q @ k.T, dim=<span class="number">1</span>) @ v</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">check_forward</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    baseline_values = py_attention(q, k, v)</span><br><span class="line">    cpp_values = attention.forward(q, k, v)[-<span class="number">1</span>]</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;base o&quot;</span>, baseline_values)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp  o&quot;</span>, cpp_values)</span><br><span class="line">    <span class="built_in">print</span>(torch.<span class="built_in">all</span>(torch.isclose(baseline_values, cpp_values)))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compare_time</span>(<span class="params">q, k, v, loop=<span class="number">100</span></span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;py&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: py_attention(q, k, v), number=loop))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: attention.forward(q, k, v), number=loop))</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    m, n = <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">    device = <span class="string">&quot;cuda&quot;</span> <span class="keyword">if</span> torch.cuda.is_available() <span class="keyword">else</span> <span class="string">&quot;cpu&quot;</span></span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;q&quot;</span>, q)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;k&quot;</span>, k)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;v&quot;</span>, v)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">20</span>)</span><br><span class="line">    check_forward(q, k, v)</span><br><span class="line">    compare_time(q, k, v)</span><br></pre></td></tr></table></figure>
+<p>测试通过后，可以再使用 <code>compare_time</code>
+函数对比一下二者的速度。理论上，二者的速度是相差无几的。因为均用的是
+PyTorch 的矩阵乘法和 softmax 函数。</p>
+<p>但是，如果需要进行更进一步的优化技巧，那么就需要自己实现矩阵乘法和
+softmax 函数了。这里，我们只实现最简单的矩阵乘法和 softmax
+函数，然后再对比一下二者的速度。</p>
+<h4 id="矩阵乘法">矩阵乘法</h4>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="function">torch::Tensor <span class="title">my_matmul</span><span class="params">(<span class="type">const</span> torch::Tensor &amp;a, <span class="type">const</span> torch::Tensor &amp;b)</span> </span>&#123;</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">dim</span>() == <span class="number">2</span> &amp;&amp; b.<span class="built_in">dim</span>() == <span class="number">2</span>, <span class="string">&quot;Input tensors must be 2-dimensional&quot;</span>);</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">size</span>(<span class="number">1</span>) == b.<span class="built_in">size</span>(<span class="number">0</span>), <span class="string">&quot;Dimensions mismatch&quot;</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> m = a.<span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">auto</span> n = b.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="keyword">auto</span> p = a.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    torch::Tensor result = torch::<span class="built_in">zeros</span>(&#123;m, n&#125;, torch::<span class="built_in">dtype</span>(torch::kFloat32));</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; m; i++) &#123;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; n; j++) &#123;</span><br><span class="line">            <span class="type">float</span> sum = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; p; k++) &#123;</span><br><span class="line">                sum += a[i][k].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;() * b[k][j].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;();</span><br><span class="line">            &#125;</span><br><span class="line">            result[i][j] = sum;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> result;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h4 id="softmax">softmax</h4>
+<p>由于
+<code>softmax</code>函数比较特殊，后续会结合算子融合一起优化，所以简单的对其进行展开。用
+<code>torch::exp</code>和
+<code>torch::sum</code>实现了一遍，为了方便也可以直接使用
+<code>torch::softmax(scores, 1)</code></p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="function">torch::Tensor <span class="title">my_softmax</span><span class="params">(<span class="type">const</span> torch::Tensor&amp; scores)</span> </span>&#123;</span><br><span class="line">    torch::Tensor exponents = torch::<span class="built_in">exp</span>(scores);</span><br><span class="line">    torch::Tensor sum = torch::<span class="built_in">sum</span>(exponents, <span class="number">1</span>, <span class="literal">true</span>);</span><br><span class="line">    <span class="keyword">return</span> exponents / sum;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>但把这两个函数替换到 <code>attention_forward</code>函数中后，再次运行
+<code>compare_time</code>函数，发现手写的 Cpp 版本的实现要比 Python
+版本的实现慢。为什么？因为，当前只是简单的实现了矩阵乘法和 softmax，而
+PyTorch 中的矩阵乘法和 softmax 都是经过优化的，所以速度会更快。</p>
+<p>另外，使用原生的矩阵乘法和 softmax 函数，可以在 GPU
+上运行，而手写的矩阵乘法和 softmax 函数，只能在 CPU
+上运行。因此，接下来将其改造为 GPU 版本，然后再进行优化。</p>
+<h3 id="gpu-版本">GPU 版本</h3>
+<h4 id="python-部分-1">Python 部分</h4>
+<p>在 <code>setup.py</code> 中更改为如下代码，把
+<code>CppExtension</code> 改为 <code>CUDAExtension</code></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> setuptools <span class="keyword">import</span> setup</span><br><span class="line"><span class="keyword">from</span> torch.utils.cpp_extension <span class="keyword">import</span> BuildExtension, CUDAExtension </span><br><span class="line"></span><br><span class="line">setup(</span><br><span class="line">    name=<span class="string">&#x27;attention&#x27;</span>,</span><br><span class="line">    ext_modules=[</span><br><span class="line">        CUDAExtension(<span class="string">&#x27;attention&#x27;</span>, [</span><br><span class="line">            <span class="string">&#x27;attention.cpp&#x27;</span>,</span><br><span class="line">            <span class="string">&#x27;attention_kernel.cu&#x27;</span>,</span><br><span class="line">        ])      </span><br><span class="line">    ],</span><br><span class="line">    cmdclass=&#123;</span><br><span class="line">        <span class="string">&#x27;build_ext&#x27;</span>: BuildExtension</span><br><span class="line">    &#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="cpp-部分-1">Cpp 部分</h4>
+<p>为了兼容之前的代码，这里将之前的 <code>attention_forward</code>更新为
+<code>attention_cpu_forward</code>，同时加了一个类型判断，如果输入的
+<code>Tensor</code>不在同一个设备上，则抛出异常。而对于
+<code>attention_cuda_forward</code>的实现需要在
+<code>attention_kernel.cu</code>中实现。注意：这里需要提前定义好
+<code>attention_cuda_forward</code>函数，否则会报错。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span></span>;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">my_matmul</span><span class="params">(<span class="type">const</span> torch::Tensor &amp;a, <span class="type">const</span> torch::Tensor &amp;b)</span> </span>&#123;</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">dim</span>() == <span class="number">2</span> &amp;&amp; b.<span class="built_in">dim</span>() == <span class="number">2</span>, <span class="string">&quot;Input tensors must be 2-dimensional&quot;</span>);</span><br><span class="line">    <span class="built_in">TORCH_CHECK</span>(a.<span class="built_in">size</span>(<span class="number">1</span>) == b.<span class="built_in">size</span>(<span class="number">0</span>), <span class="string">&quot;Dimensions mismatch&quot;</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> m = a.<span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="keyword">auto</span> n = b.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="keyword">auto</span> p = a.<span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    torch::Tensor result = torch::<span class="built_in">zeros</span>(&#123;m, n&#125;, torch::<span class="built_in">dtype</span>(torch::kFloat32));</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; m; i++) &#123;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> j = <span class="number">0</span>; j &lt; n; j++) &#123;</span><br><span class="line">            <span class="type">float</span> sum = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; p; k++) &#123;</span><br><span class="line">                sum += a[i][k].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;() * b[k][j].<span class="built_in">item</span>&lt;<span class="type">float</span>&gt;();</span><br><span class="line">            &#125;</span><br><span class="line">            result[i][j] = sum;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> result;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cpu_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line">    torch::Tensor scores = <span class="built_in">my_matmul</span>(q, k);</span><br><span class="line">    torch::Tensor attention = <span class="built_in">my_matmul</span>(torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>), v);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores, attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">// 参数：queries(Q)，keys(K)，values(V)</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor &amp;v)</span> </span>&#123;</span><br><span class="line">    <span class="keyword">if</span> (!(q.<span class="built_in">device</span>().<span class="built_in">type</span>() == k.<span class="built_in">device</span>().<span class="built_in">type</span>() &amp;&amp; q.<span class="built_in">device</span>().<span class="built_in">type</span>() == v.<span class="built_in">device</span>().<span class="built_in">type</span>())) &#123;</span><br><span class="line">        <span class="keyword">throw</span> std::<span class="built_in">runtime_error</span>(<span class="string">&quot;Input tensors q, k, and v must be on the same device&quot;</span>);</span><br><span class="line">    &#125;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (q.<span class="built_in">is_cuda</span>()) &#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">attention_cuda_forward</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>), v);</span><br><span class="line">    &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">attention_cpu_forward</span>(q, k.<span class="built_in">transpose</span>(<span class="number">0</span>, <span class="number">1</span>), v);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">PYBIND11_MODULE</span>(TORCH_EXTENSION_NAME, m) &#123;</span><br><span class="line">    m.<span class="built_in">def</span>(<span class="string">&quot;forward&quot;</span>, &amp;attention_forward, <span class="string">&quot;Attention forward (CUDA)&quot;</span>);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<h4 id="cu-部分">Cu 部分</h4>
+<p>在同级目录下，创建 <code>attention_kernel.cu</code>文件。首先实现
+<code>attention_cuda_forward</code>函数的主要逻辑。其中，主要对矩阵乘法进行了优化，使用了
+CUDA 的并行计算。然后，使用
+<code>AT_DISPATCH_FLOATING_TYPES</code>宏来实现对不同类型的支持，这样就可以支持
+<code>float</code>和 <code>double</code>类型了。</p>
+<p>对于
+<code>matrix_multiply</code>函数，与一般写法不同的是，需要提前创建好输出的
+<code>Tensor</code>，然后再传入到 CUDA 的函数中。并且需要创建好
+<code>blocks</code>和 <code>threads</code>，然后再调用 CUDA
+的函数。这里指定了每个 CUDA 是有 16 x 16
+线程的块，而这些块是可以并行计算的，所以能够加速计算。可参考 <a
+href="https://devblogs.nvidia.com/even-easier-introduction-cuda">An Even
+Easier Introduction to CUDA</a></p>
+<p>而在传递参数的时候，需要使用 <code>packed_accessor</code>。这里的
+<code>packed_accessor</code>的第一个参数是
+<code>Tensor</code>的类型，第二个参数是
+<code>Tensor</code>的维度，第三个参数是
+<code>Tensor</code>的类型，第四个参数是
+<code>Tensor</code>的维度。这里的
+<code>packed_accessor</code>的第三个参数和第四个参数，是为了支持 CUDA
+的。</p>
+<p>接下来就是实现
+<code>matrix_multiply_kernel</code>。矩阵的乘法中，如果要计算输出矩阵的第一个值，则需要用到输入矩阵的第一行和第一列。因此，这里需要根据
+<code>block</code>和
+<code>thread</code>的索引，来计算出对应的行和列。然后，就是普通的矩阵乘法的实现了。原来的矩阵乘法的实现是：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">out = [[<span class="number">0</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(m)]</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(m):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(p):</span><br><span class="line">            out[i][j] += input1[i][k] * input2[k][j]</span><br></pre></td></tr></table></figure>
+<p>相当于把外面两个循环分别交给了 CUDA 的 <code>block</code>和
+<code>thread</code>来计算。这样，就可以实现并行计算了。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiply kernel</span></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">        <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">            value += input1[row][k] * input2[k][col];</span><br><span class="line">        &#125;</span><br><span class="line">        output[row][col] = value;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_multiply</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> rows1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> cols1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> cols2 = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    torch::Tensor output = torch::<span class="built_in">zeros</span>(&#123;rows1, cols2&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((cols2 + threads.x - <span class="number">1</span>) / threads.x,</span></span></span><br><span class="line"><span class="params"><span class="function">                      (rows1 + threads.y - <span class="number">1</span>) / threads.y)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input<span class="number">1.</span><span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input<span class="number">1.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input<span class="number">2.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;());</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> output;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = <span class="built_in">matrix_multiply</span>(q, k);</span><br><span class="line">    torch::Tensor attention = <span class="built_in">matrix_multiply</span>(torch::<span class="built_in">softmax</span>(scores, <span class="number">1</span>), v);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores, attention&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<p>测试代码不变，依旧可以用上面的来进行检验。至此，最简单的实验就完成了。接下来，就是对其进行优化了。</p>
+<h2 id="矩阵乘法的优化">矩阵乘法的优化</h2>
+<h3 id="matmul">Matmul</h3>
+<p>重新回到矩阵乘法上，假设有两个矩阵 A1 和 A2，形状分别为 <span
+class="math inline">\(M \times N\)</span> 和 <span
+class="math inline">\(N \times K\)</span>，则矩阵乘法的计算公式为：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">output = torch.zeros(size=(M, K))</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(M):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(K):</span><br><span class="line">        sum_ = <span class="number">0</span></span><br><span class="line">        <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(N):</span><br><span class="line">            sum_ += A1[i][k] * A2[k][j]</span><br><span class="line">        output[i][j] = sum_</span><br></pre></td></tr></table></figure>
+<p>一种朴素的优化手段是把最后一个循环并行计算。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(M):</span><br><span class="line">    <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(K):</span><br><span class="line">        output[i][j] = <span class="built_in">sum</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: x[<span class="number">0</span>] * x[<span class="number">1</span>], <span class="built_in">zip</span>(A1[i], A2[:, j])))</span><br></pre></td></tr></table></figure>
+<p>利用多线程/进程（下文统称为job）进行并行计算可以提高程序的计算速度，但这样需要每个job都能访问到
+A1 和 A2 的数据，所以这就引入了全局内存和共享内存的概念。</p>
 <blockquote>
-<p>在含有子查询的SQL语句中,要特别注意减少对表的查询</p>
+<ul>
+<li>全局内存（Global
+Memory）：全局内存是一种在计算机程序中可被所有线程或进程访问的内存空间。它通常用于存储全局变量、静态变量以及动态分配的内存等。全局内存的特点是可以在整个程序执行过程中进行读写操作，但它的访问速度相对较慢。</li>
+<li>共享内存（Shared
+Memory）：共享内存是一种特殊的内存区域，被多个线程或进程同时访问和共享。通过将数据存储在共享内存中，不同的线程或进程可以直接读取和写入这些数据，而无需使用其他的通信机制。共享内存的特点是高效的数据共享和访问速度，因为不需要进行复制或传输数据。</li>
+</ul>
 </blockquote>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">SELECT</span> TAB_NAME <span class="keyword">FROM</span> TABLES <span class="keyword">WHERE</span> (TAB_NAME,DB_VER) <span class="operator">=</span> ( <span class="keyword">SELECT</span> TAB_NAME,DB_VER <span class="keyword">FROM</span> TAB_COLUMNS <span class="keyword">WHERE</span> VERSION <span class="operator">=</span> <span class="number">604</span>)</span><br></pre></td></tr></table></figure>
-<ol start="13" type="1">
-<li><p>通过内部函数提高SQL效率</p>
-<p>复杂的SQL往往牺牲了执行效率.
-能够掌握上面的运用函数解决问题的方法在实际工作中是非常有意义的。</p></li>
-<li><p>使用表的别名(<code>Alias</code>)：</p></li>
+<p>CUDA 中依旧存在类似的概念</p>
+<blockquote>
+<ul>
+<li>全局内存（Global Memory）：在 CUDA
+中，全局内存是一个设备（GPU）上可见的主机（CPU）内存空间。它可以由所有的线程块和线程访问，用于存储全局变量和动态分配的内存等。全局内存的读写操作相对较慢，因为涉及主机与设备之间的数据传输。</li>
+<li>共享内存（Shared Memory）：在 CUDA
+中，共享内存是位于每个线程块中的一块高速缓存内存。它被同一个线程块内的线程共享，并且比全局内存具有更快的读写速度。共享内存通常用于优化算法的性能，通过在线程块内部共享数据来减少全局内存的访问。</li>
+</ul>
+</blockquote>
+<p>简而言之，全局内存可以很方便的存各种东西，但是速度慢；共享内存是一个好东西，速度块，但通常大小受限。所以，分别有两种优化：</p>
+<ul>
+<li>全局内存中，考虑如何加速访问</li>
+<li>共享内存中，考虑如何减小占用空间</li>
+</ul>
+<p>这就引入了 Tiled matmul 算法。</p>
+<h3 id="tiled-matmul">Tiled Matmul</h3>
+<ul>
+<li>对于加速访问，在无法控制硬件的前提下，只能通过并行的方式同时读取数据。</li>
+<li>对于减小占用空间，可以通过拆分矩阵，把大矩阵拆分成若干小矩阵，然后再进行计算。</li>
+</ul>
+<p>重新思考矩阵
+<code>output</code>的计算过程，每个元素的计算其实是独立的，其本质可以拆成若干独立的小块，如下图所示：</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/setup-2d0f22ecd2e9b7c84af56792d14ba18a.gif?raw=true" />
+From https://penny-xu.github.io/blog/tiled-matrix-multiplication</p>
+<p>由于矩阵
+<code>output</code>每个元素是完全独立的，可以将其拆成若干个小矩阵来计算。如上图所示，把矩阵
+<code>output</code> 拆成了 4 个小矩阵。对应的代码如下所示：</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/block_matrix.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">block_size = <span class="number">2</span> <span class="comment"># 方便起见，这里设置为 N,M,K 的公约数。同时也拆成了 2x2 的 block</span></span><br><span class="line">block_M, block_N, block_K = M // block_size, N // block_size, K // block_size</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line">output11 = matmul(A1[:block_M, :], A2[:, :block_K])</span><br><span class="line">output12 = matmul(A1[:block_M, :], A2[:, block_K:])</span><br><span class="line">output21 = matmul(A1[block_M:, :], A2[:, :block_K])</span><br><span class="line">output22 = matmul(A1[block_M:, :], A2[:, block_K:])</span><br><span class="line">output = torch.cat([torch.cat([output11, output12], dim=<span class="number">1</span>), torch.cat([output21, output22], dim=<span class="number">1</span>)], dim=<span class="number">0</span>)</span><br><span class="line"><span class="built_in">print</span>(output)</span><br><span class="line"><span class="built_in">print</span>(A1 @ A2)</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(output, A1 @ A2)</span><br></pre></td></tr></table></figure>
+<p>对于，左上角矩阵 <code>output11</code>，实际上是由
+<code>block_size</code>个矩阵乘法，再求和得到的
+<code>output11 = matmul(A1[:block_M, :block_N], A2[:block_N, :block_K]) + matmul(A1[:block_M, block_N:], A2[block_N:, :block_K])</code>。再把它扩展的灵活一点</p>
+<ol type="1">
+<li>不局限于只能扩展为 2 <span class="math inline">\(\times\)</span> 2
+矩阵</li>
+<li>block_size 可以针对 M, N, K 进行调整</li>
 </ol>
-<pre><code>当在SQL语句中连接多个表时, 请使用表的别名并把别名前缀于每个Column上.这样一来,就可以减少解析的时间并减少那些由Column歧义引起的语法错误。</code></pre>
-<ol start="15" type="1">
-<li>识别’低效执行’的SQL语句</li>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">6</span>, <span class="number">8</span></span><br><span class="line"></span><br><span class="line">A1 = torch.rand(size=(M, N))</span><br><span class="line">A2 = torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line">block_size_M, block_size_N, block_size_K = <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span></span><br><span class="line">block_M, block_N, block_K = M // block_size_M, N // block_size_N, K // block_size_K</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">block_matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A1.shape[<span class="number">0</span>], block_M):</span><br><span class="line">        start_i, end_i = i, i + block_M</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">1</span>], block_N):</span><br><span class="line">            start_j, end_j = j, j + block_N</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">0</span>], block_K):</span><br><span class="line">                start_k, end_k = k, k + block_K</span><br><span class="line">                <span class="comment"># 计算每个 block 的矩阵乘法</span></span><br><span class="line">                sub_A1 = A1[start_i:end_i, start_k:end_k]</span><br><span class="line">                sub_A2 = A2[start_k:end_k, start_j:end_j]</span><br><span class="line">                <span class="comment"># 把每个 block 的结果放到对应的位置</span></span><br><span class="line">                output[start_i:end_i, start_j:end_j] += block_matmul(sub_A1, sub_A2)</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"><span class="built_in">print</span>(matmul(A1, A2))</span><br><span class="line"><span class="built_in">print</span>(A1 @ A2)</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(matmul(A1, A2), A1 @ A2)</span><br></pre></td></tr></table></figure>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/pytorch_cuda_ext/tmm-59dd890f48435e692c47919d0df4a5e6.gif" />
+From https://penny-xu.github.io/blog/tiled-matrix-multiplication</p>
+<p>再次回顾这样做的目的，<a
+href="https://penny-xu.github.io/blog/tiled-matrix-multiplication">原作者</a>是这么说的：</p>
+<blockquote>
+<ul>
+<li>With or without tiling, the same number of accesses into global
+memory occur. The difference is that, without tiling, each thread must
+sequentially (one after the other) access global memory 8 times.</li>
+<li>With tiling, we can parallelize the access to global memory so that
+each thread only sequentially accesses global memory 4 times.</li>
+<li>To summarize, the point is not to reduce the number of
+multiplications or even the total number of global memory accesses, but
+rather to reduce the number of sequential global memory accesses per
+thread. In other words, we better share the heavy load of memory access
+across threads.</li>
+</ul>
+</blockquote>
+<p>简而言之，作者的观点是通过拆成 block
+的形式，能够并行的读取全局内存数据。个人观点，在某些情况下，拆分后的
+block 可以恰好把数据放到共享内存中，以加速计算？</p>
+<p>最后，对于 <code>block size</code>大小的选择：</p>
+<ul>
+<li>越大的
+<code>block size</code>表示拆分后的矩阵个数变少，这样访问全局内存的次数会更多。但是，每个矩阵比较大，这样线程的并行度会更高。即，IO变小，计算变快。</li>
+<li>越小的
+<code>block size</code>表示拆分后的矩阵个数变多，这样访问全局内存的次数会更少。但是，每个矩阵比较小，这样线程的并行度会更低。即，IO变大，计算变慢。</li>
+</ul>
+<h2 id="算子融合">算子融合</h2>
+<p>算子融合是将多个计算操作合并为一个计算操作，以减少计算量和内存访问次数，从而提高计算效率。比如，矩阵乘法和
+softmax 的融合，可以减少一次内存访问。</p>
+<h3 id="softmax-的改造">softmax 的改造</h3>
+<p>softmax 的计算公式为：</p>
+<p><span class="math display">\[
+\text{softmax}(x_i) = \frac{e^{x_i}}{\sum_{j=1}^n e^{x_j}}
+\]</span></p>
+<p>而通常为了数值稳定性（避免溢出），会先计算最大值，再减去最大值，最后再计算
+softmax。即：</p>
+<p><span class="math display">\[
+\text{softmax}(x_i) = \frac{e^{x_i - \max(x)}}{\sum_{j=1}^n e^{x_j -
+\max(x)}}
+\]</span></p>
+<p>在神经网络中，其值通常为一个矩阵（不只是一个维度），所以需要对每一行进行
+softmax。示例代码如下：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">X = torch.rand(<span class="number">4</span>, <span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">my_softmax</span>(<span class="params">X, dim=<span class="number">1</span></span>):</span><br><span class="line">    X -= torch.<span class="built_in">max</span>(X, dim=dim, keepdim=<span class="literal">True</span>)[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">return</span> torch.exp(X) / torch.<span class="built_in">sum</span>(torch.exp(X), dim=dim, keepdim=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(my_softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(X, dim=<span class="number">1</span>), my_softmax(X, dim=<span class="number">1</span>))</span><br></pre></td></tr></table></figure>
+<p>但这样跟矩阵乘法进行算子融合是没有优势的，两个函数还是独立计算的。算子融合是需要糅合两种计算操作。所以，需要把
+softmax
+函数放到矩阵乘法中进行计算。即将计算过程变成一个可迭代的过程，换而言之，随着元素的增加不断更新
+softmax 的结果。</p>
+<p><a href="https://arxiv.org/pdf/1805.02867.pdf">Online normalizer
+calculation for softmax</a>
+提供了这么一种方式。简单起见，从一个元素开始介绍。现在有一个列表，里面只有一个元素
+<code>[1]</code>，其 softmax 计算过程为：</p>
+<ol type="1">
+<li>exp: 计算 <span class="math inline">\(e^{1}\)</span>，得到 <span
+class="math inline">\([e^{1}]\)</span></li>
+<li>max: 计算 <code>max(1)</code>，得到 <code>1</code></li>
+<li>-max: 计算 $e^{1 - 1} $，得到 <span class="math inline">\([e^{1 - 1}
+]\)</span></li>
+<li>sum: 计算 $e^{1- 1} $ 的和，得到 $e^{1- 1} $</li>
+<li>softmax: 计算 <span class="math inline">\([e^{1- 1} ] / (e^{1- 1}
+)\)</span>，得到 <span class="math inline">\([1]\)</span></li>
 </ol>
-<pre><code>虽然目前各种关于SQL优化的图形化工具层出不穷,但是写出自己的SQL工具来解决问题始终是一个最好的方法。</code></pre>
-<p>​<code>sql SELECT EXECUTIONS, DISK_READS, BUFFER_GETS, ROUND((BUFFER_GETS-DISK_READS)/BUFFER_GETS,2) Hit_radio, ROUND(DISK_READS/EXECUTIONS,2) Reads_per_run, SQL_TEXT FROM V$SQLAREA WHERE EXECUTIONS&gt;0 AND BUFFER_GETS &gt; 0 AND (BUFFER_GETS-DISK_READS)/BUFFER_GETS &lt; 0.8 ORDER BY 4 DESC; ​</code></p>
-<ol start="16" type="1">
-<li><p>用索引提高效率</p>
-<p>索引是表的一个概念部分,用来提高检索数据的效率，ORACLE使用了一个复杂的自平衡B-tree结构.
-通常,通过索引查询数据比全表扫描要快.
-当ORACLE找出执行查询和Update语句的最佳路径时, ORACLE优化器将使用索引.
-同样在联结多个表时使用索引也可以提高效率.
-另一个使用索引的好处是,它提供了主键(primary
-key)的唯一性验证.。那些LONG或LONG RAW数据类型, 你可以索引几乎所有的列.
-通常, 在大型表中使用索引特别有效. 当然,你也会发现,
-在扫描小表时,使用索引同样能提高效率.
-虽然使用索引能得到查询效率的提高,但是我们也必须注意到它的代价.
-索引需要空间来存储,也需要定期维护, 每当有记录在表中增减或索引列被修改时,
-索引本身也会被修改. 这意味着每条记录的INSERT , DELETE ,
-UPDATE将为此多付出4 , 5 次的磁盘I/O .
-因为索引需要额外的存储空间和处理,那些不必要的索引反而会使查询反应时间变慢.。定期的重构索引是有必要的</p></li>
+<p>现在增加一个元素，观察有哪些变化。假设增加的元素为
+2，现在有一个列表，里面有两个元素 <code>[1, 2]</code>，新增的 softmax
+计算过程为：</p>
+<ol type="1">
+<li>exp: 计算 <span class="math inline">\(e^{2}\)</span>，得到 <span
+class="math inline">\(e^{2}\)</span></li>
+<li>max: 与原来的 <code>max(1)=1</code> 比较，得到
+<code>max(1, 2)</code>，得到 <code>2</code></li>
+<li>-max：所有元素更新一遍，均减去最新的 <code>max(1, 2)=2</code>，得到
+<span class="math inline">\([e^{1- 2} , e^{2- 2} ]\)</span></li>
+<li>sum: 重新计算一遍结果</li>
+<li>softmax: 计算 <span class="math inline">\([e^{1- 2}, e^{2 - 2}] /
+(e^{1 - 2} + e^{2 - 2})\)</span></li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">ALTER</span> INDEX <span class="operator">&lt;</span>INDEXNAME<span class="operator">&gt;</span> REBUILD <span class="operator">&lt;</span>TABLESPACENAME<span class="operator">&gt;</span></span><br></pre></td></tr></table></figure>
-<ol start="17" type="1">
-<li><p>用<code>EXISTS</code>替换<code>DISTINCT</code></p>
-<p>当提交一个包含一对多表信息(比如部门表和雇员表)的查询时,避免在<code>SELECT</code>子句中使用<code>DISTINCT</code>.
-一般可以考虑用<code>EXIST</code>替换, <code>EXISTS</code>
-使查询更为迅速,因为RDBMS核心模块将在子查询的条件一旦满足后,立刻返回结果.
-例子：</p></li>
-</ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">(低效):</span><br><span class="line"><span class="keyword">SELECT</span> <span class="keyword">DISTINCT</span> DEPT_NO, DEPT_NAME <span class="keyword">FROM</span> DEPT D, EMP E <span class="keyword">WHERE</span> D.DEPT_NO <span class="operator">=</span> E.DEPT_NO</span><br><span class="line">(高效):</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_NO, DEPT_NAME <span class="keyword">FROM</span> DEPT D <span class="keyword">WHERE</span> <span class="keyword">EXISTS</span> ( <span class="keyword">SELECT</span> <span class="string">&#x27;X&#x27;</span> <span class="keyword">FROM</span> EMP E <span class="keyword">WHERE</span> E.DEPT_NO <span class="operator">=</span> D.DEPT_NO);</span><br></pre></td></tr></table></figure>
-<ol start="18" type="1">
-<li><p>sql语句用大写的</p>
-<p>因为oracle总是先解析sql语句，把小写的字母转换成大写的再执行</p></li>
-<li><p>在java代码中尽量少用连接符“＋”连接字符串！</p></li>
-<li><p>避免在索引列上使用NOT，通常我们要避免在索引列上使用NOT,
-NOT会产生在和在索引列上使用函数相同的影响.
-当ORACLE”遇到”NOT,他就会停止使用索引转而执行全表扫描。</p></li>
-<li><p>避免在索引列上使用计算</p>
-<p>WHERE子句中，如果索引列是函数的一部分．优化器将不使用索引而使用全表扫描．</p></li>
-</ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效：</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> SAL <span class="operator">*</span> <span class="number">12</span> <span class="operator">&gt;</span> <span class="number">25000</span>;</span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> SAL <span class="operator">&gt;</span> <span class="number">25000</span><span class="operator">/</span><span class="number">12</span>;</span><br></pre></td></tr></table></figure>
-<ol start="22" type="1">
-<li>用&gt;=替代&gt;</li>
+<p>对比发现，第 3 步会导致重新第 4
+步计算一遍求和结果，但这个求和结果在第 5
+步中作为分母是可以灵活调整的。原来是<span
+class="math inline">\(e^{1-1}\)</span>，现在更新为<span
+class="math inline">\(e^{1-2}+e^{2-2}\)</span>。假设原来的最大值为
+<code>old_max</code>，新的最大值为 <code>new_max</code>，原来的元素为
+<code>old_v</code>，则原来元素的 <code>exp</code>结果可以更新为</p>
+<p><span class="math display">\[
+e^{old\_v-old\_max} \rightarrow e^{old\_v-old\_max+old\_max-new\_max} =
+e^{old\_v-old\_max} \times e^{old\_max-new\_max}
+\]</span></p>
+<p>而对于新加的元素，则可以在求和部分直接加上<span
+class="math inline">\(e^{2-2}\)</span>，所以，代码如下</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">nums = [<span class="number">1</span>, <span class="number">2</span>]</span><br><span class="line">sum_, max_v = <span class="number">0.</span>, <span class="number">0.</span></span><br><span class="line">norm_v = <span class="number">0.</span></span><br><span class="line"><span class="keyword">for</span> i, num <span class="keyword">in</span> <span class="built_in">enumerate</span>(nums):</span><br><span class="line">    old_max_v = max_v</span><br><span class="line">    max_v = <span class="built_in">max</span>(max_v, num)</span><br><span class="line">    norm_v = norm_v*np.exp(old_max_v - max_v) + np.exp(num - max_v)</span><br><span class="line">softmax_nums = [np.exp(num-max_v)/ norm_v <span class="keyword">for</span> num <span class="keyword">in</span> nums] </span><br></pre></td></tr></table></figure>
+<p>对于两个元素这个公式是适用的，那么对于多个元素呢？答案自然也是同样适用的</p>
+<p><span class="math display">\[
+\begin{aligned}
+e^{v1-old\_max}+e^{v2-old\_max} &amp;\rightarrow
+e^{v1-new\_max}+e^{v2-new\_max} \\ &amp;=
+e^{v1-old\_max+old\_max-new\_max}+e^{v2-old\_max+old\_max-new\_max}
+\\  \\ &amp;= e^{v1-old\_max} \times e^{old\_max-new\_max} +
+e^{v2-old\_max} \times e^{old\_max-new\_max} \\ &amp;= (e^{v1-old\_max}
+- e^{v2-old\_max}) \times e^{old\_max-new\_max}
+\end{aligned}
+\]</span></p>
+<p>所以，将其更新为二维矩阵的形式，就有</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">X = torch.rand(<span class="number">4</span>, <span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">online_softmax</span>(<span class="params">X</span>):</span><br><span class="line">    value = torch.zeros_like(X)</span><br><span class="line">    <span class="keyword">for</span> row <span class="keyword">in</span> <span class="built_in">range</span>(X.shape[<span class="number">0</span>]):</span><br><span class="line">        row_max = <span class="number">0.0</span></span><br><span class="line">        normalizer_term = <span class="number">0.0</span></span><br><span class="line">        <span class="keyword">for</span> col <span class="keyword">in</span> <span class="built_in">range</span>(X.shape[<span class="number">1</span>]):</span><br><span class="line">            val = X[row, col]</span><br><span class="line">            old_row_max = row_max</span><br><span class="line">            row_max = <span class="built_in">max</span>(old_row_max, val)</span><br><span class="line">            normalizer_term = normalizer_term * np.exp(old_row_max - row_max) + np.exp(val - row_max)</span><br><span class="line">        value[row, :] = torch.exp(X[row, :] - row_max) / normalizer_term</span><br><span class="line">    <span class="keyword">return</span> value</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(X, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(online_softmax(X))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(X, dim=<span class="number">1</span>), online_softmax(X))</span><br></pre></td></tr></table></figure>
+<h3 id="softmax-矩阵乘法">softmax + 矩阵乘法</h3>
+<p>一般的融合操作是需要在矩阵乘法后，计算结果的 softmax。即</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line">output = torch.softmax(A1 @ A2, dim=<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
+<p>在结合上面的内容后，将其改造为</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">2</span>, <span class="number">4</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">matmul_softmax</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(A1.shape[<span class="number">0</span>]):</span><br><span class="line">        row_max = <span class="number">0.0</span></span><br><span class="line">        normalizer_term = <span class="number">0.0</span>  </span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(A2.shape[<span class="number">1</span>]):</span><br><span class="line">            val = output[i, j] = <span class="built_in">sum</span>(<span class="built_in">map</span>(<span class="keyword">lambda</span> x: x[<span class="number">0</span>] * x[<span class="number">1</span>], <span class="built_in">zip</span>(A1[i], A2[:, j])))</span><br><span class="line">          </span><br><span class="line">            old_row_max = row_max</span><br><span class="line">            row_max = <span class="built_in">max</span>(old_row_max, val)</span><br><span class="line">            normalizer_term = normalizer_term * np.exp(old_row_max - row_max) + np.exp(val - row_max)</span><br><span class="line">        output[i, :] = torch.exp(output[i, :] - row_max) / normalizer_term</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(A1 @ A2, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(matmul_softmax(A1, A2))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(A1 @ A2, dim=<span class="number">1</span>), matmul_softmax(A1, A2))</span><br></pre></td></tr></table></figure>
+<p>更进一步，结合 Tiled matmul，将其改造为：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"></span><br><span class="line">M, N, K = <span class="number">4</span>, <span class="number">6</span>, <span class="number">8</span></span><br><span class="line">A1, A2 = torch.rand(size=(M, N)), torch.rand(size=(N, K))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">block_matmul</span>(<span class="params">sub_A1, sub_A2</span>):</span><br><span class="line">    output = torch.zeros(size=(sub_A1.shape[<span class="number">0</span>], sub_A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(sub_A1.shape[<span class="number">0</span>]):</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">1</span>]):</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(sub_A2.shape[<span class="number">0</span>]):</span><br><span class="line">                output[i][j] += sub_A1[i][k] * sub_A2[k][j]            </span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">tiled_matmul_softmax</span>(<span class="params">A1, A2</span>):</span><br><span class="line">    block_size_M, block_size_N, block_size_K = <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span></span><br><span class="line">    block_M, block_N, block_K = M // block_size_M, N // block_size_N, K // block_size_K</span><br><span class="line"></span><br><span class="line">    output = torch.zeros(size=(A1.shape[<span class="number">0</span>], A2.shape[<span class="number">1</span>]))</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A1.shape[<span class="number">0</span>], block_M):</span><br><span class="line">        start_i, end_i = i, i + block_M</span><br><span class="line">        row_max = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line">        old_row_max = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line">        normalizer_term = torch.tensor([[<span class="number">0.</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_N)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(block_M)])</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">1</span>], block_N):</span><br><span class="line">            start_j, end_j = j, j + block_N</span><br><span class="line">            <span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, A2.shape[<span class="number">0</span>], block_K):</span><br><span class="line">                start_k, end_k = k, k + block_K</span><br><span class="line">                sub_A1 = A1[start_i:end_i, start_k:end_k]</span><br><span class="line">                sub_A2 = A2[start_k:end_k, start_j:end_j]</span><br><span class="line">                output[start_i:end_i, start_j:end_j] += block_matmul(sub_A1, sub_A2)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># 这里算完了每个block的结果，所以需要将其拆分成每个block，然后再计算softmax</span></span><br><span class="line">            <span class="keyword">for</span> ii, row <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_i, end_i)):            </span><br><span class="line">                <span class="keyword">for</span> jj, col <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_j, end_j)):</span><br><span class="line">                    val = output[row][col]</span><br><span class="line">                    old_row_max[ii][jj] = row_max[ii][jj]</span><br><span class="line">                    row_max[ii][jj] = <span class="built_in">max</span>(old_row_max[ii][jj], val)</span><br><span class="line">                    normalizer_term[ii][jj] = normalizer_term[ii][jj] * np.exp(old_row_max[ii][jj] - row_max[ii][jj]) + np.exp(val - row_max[ii][jj])</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> ii, row <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">range</span>(start_i, end_i)):</span><br><span class="line">            row_max_v, _ = torch.<span class="built_in">max</span>(row_max, dim=<span class="number">1</span>)</span><br><span class="line">            <span class="comment"># 重算 sum, 代入公式 old_v*exp(old_max - new_max)</span></span><br><span class="line">            sum_ = torch.<span class="built_in">sum</span>(normalizer_term[ii] * torch.exp(row_max[ii] - row_max_v[ii]))</span><br><span class="line">            output[row, :] = torch.exp(output[row, :] - row_max_v[ii]) / sum_</span><br><span class="line">    <span class="keyword">return</span> output</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.softmax(A1 @ A2, dim=<span class="number">1</span>))</span><br><span class="line"><span class="built_in">print</span>(tiled_matmul_softmax(A1, A2))</span><br><span class="line"><span class="keyword">assert</span> torch.allclose(torch.softmax(A1 @ A2, dim=<span class="number">1</span>), tiled_matmul_softmax(A1, A2))</span><br></pre></td></tr></table></figure>
+<p>由于这里是每次计算出来是一个 block ，所以要把 block
+拆分出每个元素，计算 block 中每行每列的最大值 row_max 以及分母
+normalizer_term。</p>
+<p>最后在计算 softmax 时，要计算所有 block 的最大值
+<code>torch.max(row_max, dim=1)</code>，还需要计算分母，这里需要考虑所有的
+block，可以类比于合并计算 <code>[1, 2]</code>,
+<code>[3, 4]</code>两个列表的normalizer_term 。已知对应的
+normalizer_term <span class="math inline">\(e^{1-2}+e^{2-2}\)</span> 和
+<span class="math inline">\(e^{3-4}+e^{3-4}\)</span>，合并后的结果应当是
+<span
+class="math inline">\(e^{1-4}+e^{2-4}+e^{3-4}+e^{3-4}\)</span>。将其公式化写作：</p>
+<p><span class="math display">\[
+\begin{aligned}
+\sum_{i=1}^{m} e^{x_i-max(x)} + \sum_{i=1}^{n} e^{y_i-max(y)} &amp;
+\rightarrow \sum_{i=1}^{m} e^{x_i-max(x,y)} + \sum_{i=1}^{n}
+e^{y_i-max(x,y)} \\ &amp;= \sum_{i=1}^{m}
+e^{x_i-max(x)}e^{max(x)-max(x,y)} + \sum_{i=1}^{n}
+e^{y_i-max(x,y)}e^{max(y)-max(x,y)}
+\end{aligned}
+\]</span></p>
+<p>假设 <code>max(x,y)=max(x)</code>，那么对应项将乘
+1，这并不会对结果有任何影响。对应的代码为：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">sum_ = torch.<span class="built_in">sum</span>(normalizer_term[ii] * torch.exp(row_max[ii] - row_max_v[ii]))</span><br></pre></td></tr></table></figure>
+<p>至此，完成了矩阵乘法和 softmax 的融合。接下来，会实现 cuda
+版本以对比性能。</p>
+<h3 id="cuda-实现">Cuda 实现</h3>
+<p>首先，接着之前矩阵乘法的cuda实现，</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">    <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">    <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">        value += input1[row][k] * input2[k][col];</span><br><span class="line">    &#125;</span><br><span class="line">    output[row][col] = value</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>在循环之后，已经计算完了输出矩阵中的一项。需要继续算每行的
+<code>row_max</code>和 <code>normalizer_term</code>。但由于这里是 cuda
+中的某个 block，所以需要借助共享内存来通信每行的结果。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="comment">// 使用共享内存，计算每个 row 的最大值</span></span><br><span class="line">__shared__ <span class="type">scalar_t</span> row_max[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">__shared__ <span class="type">scalar_t</span> normalizer_term[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">row_max[threadIdx.y][threadIdx.x] = value; <span class="comment">// 先把计算结果放到 row_max 中，以便于比较大小</span></span><br><span class="line">__syncthreads(); <span class="comment">// 这行代码是为了保证每个线程都已经计算完了，才能进行下一步的操作</span></span><br></pre></td></tr></table></figure>
+<p>计算过程分为三个步骤：1. 找到每行的最大值</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">    <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">        row_max[threadIdx.y][threadIdx.x] = <span class="built_in">max</span>(row_max[threadIdx.y][threadIdx.x], row_max[threadIdx.y][threadIdx.x + i]);</span><br><span class="line">    &#125;</span><br><span class="line">    __syncthreads();</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<ol start="2" type="1">
+<li>计算每行的 softmax 的分母每项组成</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> DEPTNO <span class="operator">&gt;=</span> <span class="number">4</span></span><br><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> DEPTNO <span class="operator">&gt;</span> <span class="number">3</span></span><br></pre></td></tr></table></figure>
-<ol start="23" type="1">
-<li><p>用UNION替换OR (适用于索引列)</p>
-<p>通常情况下, 用UNION替换WHERE子句中的OR将会起到较好的效果.
-对索引列使用OR将造成全表扫描. 注意, 以上规则只针对多个索引列有效.
-如果有column没有被索引, 查询效率可能会因为你没有选择OR而降低.
-在下面的例子中, LOC_ID 和REGION上都建有索引.</p></li>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line">normalizer_term[threadIdx.y][threadIdx.x] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]);</span><br><span class="line">__syncthreads();</span><br></pre></td></tr></table></figure>
+<ol start="3" type="1">
+<li>计算每行的 softmax 的每项之后</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span></span><br><span class="line"><span class="keyword">UNION</span></span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> REGION <span class="operator">=</span> &quot;MELBOURNE&quot;</span><br><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span> <span class="keyword">OR</span> REGION <span class="operator">=</span> &quot;MELBOURNE&quot;</span><br></pre></td></tr></table></figure>
-<ol start="24" type="1">
-<li><p>用IN来替换OR</p>
-<p>这是一条简单易记的规则，但是实际的执行效果还须检验，在ORACLE8下，两者的执行路径似乎是相同的．</p></li>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">    <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">        normalizer_term[threadIdx.y][threadIdx.x] += normalizer_term[threadIdx.y][threadIdx.x + i];</span><br><span class="line">    &#125;</span><br><span class="line">    __syncthreads();</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// 最后将其更新到输出矩阵中</span></span><br><span class="line">output[row][col] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]) / normalizer_term[threadIdx.y][<span class="number">0</span>];</span><br></pre></td></tr></table></figure>
+<p>完整代码如下（这里没有实现完整的attention，仅仅是一个矩阵乘法 +
+softmax 计算）：</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="comment">// Matrix multiply kernel</span></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> row = blockIdx.y * blockDim.y + threadIdx.y;</span><br><span class="line">    <span class="type">int</span> col = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>) &amp;&amp; col &lt; input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>)) &#123;</span><br><span class="line">        <span class="type">scalar_t</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>); ++k) &#123;</span><br><span class="line">            value += input1[row][k] * input2[k][col];</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 使用共享内存，计算每个 row 的最大值</span></span><br><span class="line">        __shared__ <span class="type">scalar_t</span> row_max[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">        __shared__ <span class="type">scalar_t</span> normalizer_term[<span class="number">16</span>][<span class="number">16</span>];</span><br><span class="line">        row_max[threadIdx.y][threadIdx.x] = value;</span><br><span class="line">        __syncthreads();</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">            <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">                row_max[threadIdx.y][threadIdx.x] = <span class="built_in">max</span>(row_max[threadIdx.y][threadIdx.x], row_max[threadIdx.y][threadIdx.x + i]);</span><br><span class="line">            &#125;</span><br><span class="line">            __syncthreads();</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="comment">// 计算每个 row 的 softmax 的分母</span></span><br><span class="line">        normalizer_term[threadIdx.y][threadIdx.x] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]);</span><br><span class="line"></span><br><span class="line">        __syncthreads();</span><br><span class="line">        <span class="comment">// 计算每个 row  normalizer_term之和</span></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = blockDim.x / <span class="number">2</span>; i &gt; <span class="number">0</span>; i /= <span class="number">2</span>) &#123;</span><br><span class="line">            <span class="keyword">if</span> (threadIdx.x &lt; i) &#123;</span><br><span class="line">                normalizer_term[threadIdx.y][threadIdx.x] += normalizer_term[threadIdx.y][threadIdx.x + i];</span><br><span class="line">            &#125;</span><br><span class="line">            __syncthreads();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="comment">// 计算每个 row 的 softmax</span></span><br><span class="line">        output[row][col] = <span class="built_in">exp</span>(value - row_max[threadIdx.y][<span class="number">0</span>]) / normalizer_term[threadIdx.y][<span class="number">0</span>];</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_multiply</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> rows1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> cols1 = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> cols2 = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    torch::Tensor output = torch::<span class="built_in">zeros</span>(&#123;rows1, cols2&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(<span class="number">16</span>, <span class="number">16</span>)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((cols2 + threads.x - <span class="number">1</span>) / threads.x,</span></span></span><br><span class="line"><span class="params"><span class="function">                      (rows1 + threads.y - <span class="number">1</span>) / threads.y)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input<span class="number">1.</span><span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input<span class="number">1.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input<span class="number">2.</span><span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">2</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;());</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> output;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">// 这里偷懒没有换名字</span></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    torch::Tensor scores = <span class="built_in">matrix_multiply</span>(q, k);</span><br><span class="line">    <span class="keyword">return</span> &#123;scores&#125;;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>测试代码如下</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> mulsoftmax</span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line">seed = <span class="number">42</span></span><br><span class="line">torch.manual_seed(seed)</span><br><span class="line">torch.cuda.manual_seed(seed)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">my_py_softmax</span>(<span class="params">x, dim</span>):</span><br><span class="line">    e = torch.exp(x)</span><br><span class="line">    s = torch.<span class="built_in">sum</span>(e, dim=dim, keepdim=<span class="literal">True</span>)</span><br><span class="line">    <span class="keyword">return</span> e / s</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">py_mulsoft</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    <span class="comment"># print(q@k.T)</span></span><br><span class="line">    <span class="keyword">return</span> torch.softmax(q @ k.T, dim=<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">check_forward</span>(<span class="params">q, k, v</span>):</span><br><span class="line">    baseline_values = py_mulsoft(q, k, v)</span><br><span class="line">    cpp_values = mulsoftmax.forward(q, k, v)[<span class="number">0</span>]</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;base o&quot;</span>, baseline_values)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp  o&quot;</span>, cpp_values)</span><br><span class="line">    <span class="built_in">print</span>(torch.<span class="built_in">all</span>(torch.isclose(baseline_values, cpp_values)))</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compare_time</span>(<span class="params">loop=<span class="number">100</span></span>):</span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;py&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: py_mulsoft(q, k, v), number=loop))</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;cpp&quot;</span>, timeit.timeit(<span class="keyword">lambda</span>: mulsoftmax.forward(q, k, v)[<span class="number">0</span>], number=loop))</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    m, n = <span class="number">16</span>, <span class="number">40</span></span><br><span class="line">    device = <span class="string">&quot;cuda&quot;</span></span><br><span class="line">    q, k, v = torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device), torch.rand(size=(m, n), device=device)</span><br><span class="line">    <span class="comment"># 先检查结果是否正确</span></span><br><span class="line">    check_forward(q, k, v)</span><br><span class="line">    q, k, v = torch.rand(size=(m, n)), torch.rand(size=(m, n)), torch.rand(size=(m, n))</span><br><span class="line">    <span class="comment"># 循环1w次，对比性能差距</span></span><br><span class="line">    compare_time(<span class="number">10000</span>)</span><br></pre></td></tr></table></figure>
+<p>输出结果如下：</p>
+<table>
+<thead>
+<tr>
+<th></th>
+<th>py</th>
+<th>cuda</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>0</td>
+<td>0.5136</td>
+<td>0.2909</td>
+</tr>
+<tr>
+<td>1</td>
+<td>0.6143</td>
+<td>0.3322</td>
+</tr>
+<tr>
+<td>2</td>
+<td>0.7300</td>
+<td>0.3608</td>
+</tr>
+</tbody>
+</table>
+<h3 id="其他实现">其他实现</h3>
+<h4
+id="二维矩阵转一维矩阵实现矩阵乘法">二维矩阵转一维矩阵，实现矩阵乘法</h4>
+<p>在某些情况下，为了降低空间复杂度，会把二维矩阵展开为一维矩阵，再进行矩阵乘法。这里在这个基础上，再实现了
+softmax
+的计算。其本质上是把二维矩阵转为一维矩阵，再进行矩阵乘法。原来的<code>input1[row][k]</code>
+-&gt; <code>input1[row * K + k]</code>，<code>input2[k][col]</code>
+-&gt; <code>input2[k * N + col]</code>。但这里暂时不好实现 softmax
+融合，因为 softmax
+需要计算每行的最大值，这里的一维矩阵无法直接计算每行的最大值。所以这里偷懒先使用判断的方法，如果是最后一列，则计算
+softmax。因此，导致算的速度会比较慢。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;torch/extension.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda.h&gt;</span></span></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;cuda_runtime.h&gt;</span></span></span><br><span class="line"></span><br><span class="line"><span class="meta">#<span class="keyword">include</span> <span class="string">&lt;vector&gt;</span></span></span><br><span class="line"></span><br><span class="line">#DEFINE BLOCK_SIZE <span class="number">256</span>;</span><br><span class="line"></span><br><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">matrix_multiply_vector_kernel</span><span class="params">(<span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input1,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; input2,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output,</span></span></span><br><span class="line"><span class="params"><span class="function">                                       <span class="type">const</span> <span class="type">int</span> M, <span class="type">const</span> <span class="type">int</span> N, <span class="type">const</span> <span class="type">int</span> K</span></span></span><br><span class="line"><span class="params"><span class="function">                                       )</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> index = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> row = index / N;</span><br><span class="line">    <span class="type">int</span> col = index % N;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; M &amp;&amp; col &lt; N) &#123;</span><br><span class="line">        <span class="type">float</span> value = <span class="number">0.0</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> k = <span class="number">0</span>; k &lt; K; ++k) &#123;</span><br><span class="line">            value += input1[row * K + k] * input2[k * N + col];</span><br><span class="line">        &#125;</span><br><span class="line">        output[row * N + col] = value;</span><br><span class="line">        <span class="keyword">if</span> (col == N - <span class="number">1</span>) &#123;</span><br><span class="line">            <span class="type">float</span> row_max = <span class="number">0.0</span>;</span><br><span class="line">            <span class="type">float</span> normalizer_term = <span class="number">0.0</span>;        </span><br><span class="line">            <span class="type">float</span> old_row_max = <span class="number">0.0</span>;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">                old_row_max = row_max;</span><br><span class="line">                row_max = <span class="built_in">max</span>(row_max, output[row * N + i]);</span><br><span class="line">                normalizer_term = normalizer_term * <span class="built_in">exp</span>(old_row_max - row_max) + <span class="built_in">exp</span>(output[row * N + i] - row_max);</span><br><span class="line">            &#125;</span><br><span class="line">            <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">                output[row * N + i] = <span class="built_in">exp</span>(output[row * N + i] - row_max) / normalizer_term;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">matmul_vector</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> M = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> K = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> N = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(BLOCK_SIZE)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((M * N + threads.x - <span class="number">1</span>) / threads.x)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Reshape input tensors to vectors</span></span><br><span class="line">    <span class="keyword">auto</span> input1_vector = input<span class="number">1.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);</span><br><span class="line">    <span class="keyword">auto</span> input2_vector = input<span class="number">2.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);    </span><br><span class="line">    torch::Tensor output_vector = torch::<span class="built_in">zeros</span>(&#123;M * N&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input1_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_vector_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_vector_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input1_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input2_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N, K</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line">    <span class="keyword">return</span> &#123;output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;), output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;)&#125;;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">std::vector&lt;torch::Tensor&gt; <span class="title">attention_cuda_forward</span><span class="params">(</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor q,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor k,</span></span></span><br><span class="line"><span class="params"><span class="function">    torch::Tensor v)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">matmul_vector</span>(q, k);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>输出结果如下：</p>
+<table>
+<thead>
+<tr>
+<th></th>
+<th>py</th>
+<th>cuda</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>0</td>
+<td>0.4239</td>
+<td>0.4907</td>
+</tr>
+<tr>
+<td>1</td>
+<td>0.5069</td>
+<td>0.4388</td>
+</tr>
+<tr>
+<td>2</td>
+<td>0.5462</td>
+<td>0.5799</td>
+</tr>
+</tbody>
+</table>
+<p>尽管用了一种比较笨的方法，但是速度实际上与原生的 pytorch
+相差无几。</p>
+<h4 id="单独实现-softmax">单独实现 softmax</h4>
+<p>相较于非算子融合的写法，这里实现了一维矩阵的
+softmax。相当于在计算矩阵乘法后再算 softmax。比较朴素的写法。。。</p>
+<figure class="highlight cpp"><table><tr><td class="code"><pre><span class="line"><span class="keyword">template</span> &lt;<span class="keyword">typename</span> <span class="type">scalar_t</span>&gt;</span><br><span class="line"><span class="function">__global__ <span class="type">void</span> <span class="title">softmax_kernel</span><span class="params">(torch::PackedTensorAccessor&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt; output,</span></span></span><br><span class="line"><span class="params"><span class="function">                                 <span class="type">const</span> <span class="type">int</span> M, <span class="type">const</span> <span class="type">int</span> N)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> index = blockIdx.x * blockDim.x + threadIdx.x;</span><br><span class="line">    <span class="type">int</span> row = index / N;</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (row &lt; M) &#123;</span><br><span class="line">        <span class="type">float</span> row_max = <span class="number">0.0</span>;</span><br><span class="line">        <span class="type">float</span> normalizer_term = <span class="number">0.0</span>;        </span><br><span class="line">        <span class="type">float</span> old_row_max = <span class="number">0.0</span>;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">            old_row_max = row_max;</span><br><span class="line">            row_max = <span class="built_in">max</span>(row_max, output[row * N + i]);</span><br><span class="line">            normalizer_term = normalizer_term * <span class="built_in">exp</span>(old_row_max - row_max) + <span class="built_in">exp</span>(output[row * N + i] - row_max);</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="type">int</span> i = <span class="number">0</span>; i &lt; N; ++i) &#123;</span><br><span class="line">            output[row * N + i] = <span class="built_in">exp</span>(output[row * N + i] - row_max) / normalizer_term;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="function">torch::Tensor <span class="title">matrix_softmax_vector_softmax</span><span class="params">(torch::Tensor input1, torch::Tensor input2)</span> </span>&#123;</span><br><span class="line">    <span class="type">int</span> M = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">0</span>);</span><br><span class="line">    <span class="type">int</span> K = input<span class="number">1.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line">    <span class="type">int</span> N = input<span class="number">2.</span><span class="built_in">size</span>(<span class="number">1</span>);</span><br><span class="line"></span><br><span class="line">    <span class="keyword">auto</span> options = torch::<span class="built_in">TensorOptions</span>().<span class="built_in">device</span>(input<span class="number">1.</span><span class="built_in">device</span>());</span><br><span class="line">    </span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">threads</span><span class="params">(BLOCK_SIZE_VECTOR)</span></span>;</span><br><span class="line">    <span class="function"><span class="type">const</span> dim3 <span class="title">blocks</span><span class="params">((M * N + threads.x - <span class="number">1</span>) / threads.x)</span></span>;</span><br><span class="line"></span><br><span class="line">    <span class="comment">// Reshape input tensors to vectors</span></span><br><span class="line">    <span class="keyword">auto</span> input1_vector = input<span class="number">1.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);</span><br><span class="line">    <span class="keyword">auto</span> input2_vector = input<span class="number">2.</span><span class="built_in">reshape</span>(&#123;<span class="number">-1</span>&#125;);    </span><br><span class="line">    torch::Tensor output_vector = torch::<span class="built_in">zeros</span>(&#123;M * N&#125;, options);</span><br><span class="line"></span><br><span class="line">    <span class="comment">// 普通一维的矩阵乘法</span></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(input1_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;matrix_multiply_vector_softmax_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        matrix_multiply_vector_softmax_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            input1_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            input2_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N, K</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line"></span><br><span class="line">    <span class="built_in">cudaDeviceSynchronize</span>();</span><br><span class="line"></span><br><span class="line">    <span class="built_in">AT_DISPATCH_FLOATING_TYPES</span>(output_vector.<span class="built_in">scalar_type</span>(), <span class="string">&quot;softmax_kernel&quot;</span>, ([&amp;] &#123;</span><br><span class="line">        softmax_kernel&lt;&lt;&lt;blocks, threads&gt;&gt;&gt;(</span><br><span class="line">            output_vector.<span class="built_in">packed_accessor</span>&lt;<span class="type">scalar_t</span>,<span class="number">1</span>,torch::RestrictPtrTraits,<span class="type">size_t</span>&gt;(),</span><br><span class="line">            M, N</span><br><span class="line">        );</span><br><span class="line">    &#125;));</span><br><span class="line">    <span class="keyword">return</span> output_vector.<span class="built_in">reshape</span>(&#123;M, N&#125;);</span><br><span class="line"></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>输出结果如下：</p>
+<table>
+<thead>
+<tr>
+<th></th>
+<th>py</th>
+<th>cuda</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>0</td>
+<td>0.4239</td>
+<td>1.1874</td>
+</tr>
+<tr>
+<td>1</td>
+<td>0.5069</td>
+<td>1.1101</td>
+</tr>
+<tr>
+<td>2</td>
+<td>0.5462</td>
+<td>1.3238</td>
+</tr>
+</tbody>
+</table>
+<p>对比发现，这里的cuda还是会比pytorch的慢一些，这是因为没有用自带的softmax，且没有使用算子融合。所以就算速度会更慢。</p>
+<h2 id="参考">参考</h2>
+<p>[1] <a
+href="https://pytorch.org/tutorials/advanced/cpp_extension.html">https://pytorch.org/tutorials/advanced/cpp_extension.html</a>。</p>
+<p>[2] <a
+href="https://github.com/ELS-RD/kernl/blob/main/tutorial/1%20-%20tiled%20matmul.ipynb">Tiled
+matmul</a></p>
+<p>[3] <a
+href="https://github.com/ELS-RD/kernl/blob/main/tutorial/3%20-%20online%20softmax.ipynb">Online
+softmax</a></p>
+<p>[4] <a href="https://arxiv.org/pdf/1805.02867.pdf">Online normalizer
+calculation for softmax</a></p>
+]]></content>
+      <categories>
+        <category>DeepLearning</category>
+      </categories>
+      <tags>
+        <tag>CUDA</tag>
+        <tag>PyTorch</tag>
+        <tag>Cpp</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>交叉熵的反向传播 Step by Step（PyTorch）</title>
+    <url>/hexo_blog/2023/09/03/PyTorch/%E4%BA%A4%E5%8F%89%E7%86%B5%E7%9A%84%E5%8F%8D%E5%90%91%E4%BC%A0%E6%92%AD%20Step%20by%20Step/</url>
+    <content><![CDATA[<p>本文从公式开始，一步步用PyTorch实现自定义的交叉熵损失函数，最后理解分布式损失函数</p>
+<span id="more"></span>
+<h2 id="step-1a-simple-case">Step 1：A Simple Case</h2>
+<p>以交叉熵为例，普通的二分类问题 <span class="math display">\[
+\text {Binary Cross Entropy} = -\left (y \log (\hat {y}) + (1-y) \log
+(1-\hat {y})\right)
+\]</span> 更常见的多分类问题 <span class="math display">\[
+\text {Multiclass Cross Entropy} = -\sum_{c=0}^{C} y_c \log (\hat {y}_c)
+\]</span> 其中，<span class="math inline">\(y\)</span>
+表示真实的标签值（通常是one-hot向量），<span
+class="math inline">\(\hat{y}\)</span> 表示模型的输出，<span
+class="math inline">\(C\)</span> 表示有多少类别需要分类</p>
+<p>在 PyTorch 中调用（本文使用的torch version==2.0.0）</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">criterion = torch.nn.CrossEntropyLoss()</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])   <span class="comment"># \hat&#123;y&#125; Shape: bs, C</span></span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])                 <span class="comment"># y       Shape: bs</span></span><br><span class="line"></span><br><span class="line">loss = criterion(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398)</span></span><br></pre></td></tr></table></figure>
+<p>这里的<code>outputs</code>是模型的输出，<code>targets</code>是真实的标签值</p>
+<h2 id="step-2-手写">Step 2: 手写</h2>
+<p>实现<code>CrossEntropyLoss</code>需要两个条件</p>
+<ol type="1">
+<li>使用 PyTorch 写出损失函数的公式，以及对应的导数</li>
+<li>根据公式计算loss，并计算梯度（grad）进行反向传播</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span>…. <span class="keyword">FROM</span> LOCATION <span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span> <span class="keyword">OR</span> LOC_ID <span class="operator">=</span> <span class="number">20</span> <span class="keyword">OR</span> LOC_ID <span class="operator">=</span> <span class="number">30</span></span><br><span class="line">高效</span><br><span class="line"><span class="keyword">SELECT</span>… <span class="keyword">FROM</span> LOCATION <span class="keyword">WHERE</span> LOC_IN <span class="keyword">IN</span> (<span class="number">10</span>,<span class="number">20</span>,<span class="number">30</span>);</span><br></pre></td></tr></table></figure>
-<ol start="25" type="1">
-<li><p>避免在索引列上使用IS NULL和IS NOT NULL</p>
-<p>避免在索引中使用任何可以为空的列，ORACLE将无法使用该索引．对于单列索引，如果列包含空值，索引中将不存在此记录.
-对于复合索引，如果每个列都为空，索引中同样不存在此记录.
-如果至少有一个列不为空，则记录存在于索引中．举例:
-如果唯一性索引建立在表的A列和B列上,
-并且表中存在一条记录的A,B值为(123,null) ,
-ORACLE将不接受下一条具有相同A,B值（123,null）的记录(插入).
-然而如果所有的索引列都为空，ORACLE将认为整个键值为空而空不等于空.
-因此你可以插入1000 条具有相同键值的记录,当然它们都是空!
-因为空值不存在于索引列中,所以WHERE子句中对索引列进行空值比较将使ORACLE停用该索引.</p></li>
+<h3 id="公式">公式</h3>
+<p>从<span class="math inline">\(-\sum_{c=0}^{C} y_c \log (\hat
+{y}_c)\)</span>出发</p>
+<ol type="1">
+<li>由于模型输出并不是单纯的概率值，因此会对模型输出进行<code>softmax</code>计算，得到<span
+class="math inline">\(\hat{y}\)</span> <span
+class="math display">\[\text {softmax}(x_i) = \frac
+{e^{x_i}}{\sum_{j=1}^n e^{x_j}}\]</span></li>
+<li>再以简单的单分类任务来说，
+<ol type="1">
+<li><span class="math inline">\(y\)</span> = [1, 0, 0], <span
+class="math inline">\(\hat{y}\)</span> = [0.5, 0.2, 0.3] -&gt; log(<span
+class="math inline">\(\hat{y}\)</span>)=[-0.9398, -1.2398, -1.1398]</li>
+<li>计算时，只有c=0时，结果有值（-0.9398），其他类别时为0，无需计算</li>
+<li>因此，实际最终结果就是取出对应类别的 <span
+class="math inline">\(\hat{y}\)</span> 的log值</li>
+</ol></li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效: (索引失效)</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPARTMENT <span class="keyword">WHERE</span> DEPT_CODE <span class="keyword">IS</span> <span class="keyword">NOT</span> <span class="keyword">NULL</span>;</span><br><span class="line">高效: (索引有效)</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPARTMENT <span class="keyword">WHERE</span> DEPT_CODE <span class="operator">&gt;=</span><span class="number">0</span>;</span><br></pre></td></tr></table></figure>
-<ol start="26" type="1">
-<li><p>总是使用索引的第一个列</p>
-<p>如果索引是建立在多个列上, 只有在它的第一个列(leading
-column)被where子句引用时,优化器才会选择使用该索引.
-这也是一条简单而重要的规则，当仅引用索引的第二个列时,优化器使用了全表扫描而忽略了索引。</p></li>
-<li><p>用UNION-ALL 替换UNION ( 如果有可能的话)</p>
-<p>当SQL
-语句需要UNION两个查询结果集合时,这两个结果集合会以UNION-ALL的方式被合并,
-然后在输出最终结果前进行排序. 如果用UNION ALL替代UNION,
-这样排序就不是必要了. 效率就会因此得到提高. 需要注意的是，UNION ALL
-将重复输出两个结果集合中相同记录. 因此各位还是要从业务需求分析使用UNION
-ALL的可行性. UNION
-将对结果集合排序,这个操作会使用到SORT_AREA_SIZE这块内存.
-对于这块内存的优化也是相当重要的.
-下面的SQL可以用来查询排序的消耗量</p></li>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">MyCrossEntropyLoss</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># outputs.shape: bs, C</span></span><br><span class="line">    <span class="comment"># targets.shape: bs</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  <span class="comment"># 对预测结果进行softmax操作</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 使用索引选择对应类别的概率值，并使用负对数函数计算损失</span></span><br><span class="line">    loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) <span class="comment"># 对每个计算取对应的target</span></span><br><span class="line">    <span class="keyword">return</span> loss.mean() <span class="comment"># 最终返回结果的平均值</span></span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])	<span class="comment"># \hat&#123;y&#125; Shape: bs, C</span></span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])	</span><br><span class="line"></span><br><span class="line">loss = MyCrossEntropyLoss(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398)</span></span><br></pre></td></tr></table></figure>
+<h4 id="梯度">梯度</h4>
+<ol type="1">
+<li><p>结合前面的计算，每次计算loss时，实际上只有对应类别的计算是有效的，因此将其简化为<span
+class="math inline">\(-y_c\log(\hat{y}_c)\)</span>，而由于<span
+class="math inline">\(y_c=1\)</span>，所以其实只有<span
+class="math inline">\(-\log(\hat{y}_c)\)</span></p></li>
+<li><p>这个公式只需要计算log的导数。对于<span
+class="math inline">\(f(x)=\log(x)\)</span>，其导数<span
+class="math inline">\(f^{&#39;}(x)=\frac{1}{x}\)</span>。</p></li>
+<li><p>因此，这里的梯度很简单，如下所示，其中，<span
+class="math inline">\(y_c=1\)</span> <span class="math display">\[
+\frac {\partial \text {CrossEntropy}(y, \hat {y})}{\partial \hat {y}_c}
+= -\frac {y_c}{\hat {y}_c}=-\frac{1}{\hat{y}_c}
+\]</span></p></li>
+<li><p>而对于模型的输出而言，还有一个SoftMax计算。SoftMax的梯度有</p>
+<ol type="1">
+<li>当 <span class="math inline">\(i = j\)</span> 时：</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效：</span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line"><span class="keyword">UNION</span></span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line"><span class="keyword">UNION</span> <span class="keyword">ALL</span></span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br></pre></td></tr></table></figure>
-<ol start="28" type="1">
-<li><p>用WHERE替代ORDER BY</p>
-<p>ORDER BY 子句只在两种严格的条件下使用索引.</p>
-<p>ORDER BY中所有的列必须包含在相同的索引中并保持在索引中的排列顺序.</p>
-<p>ORDER BY中所有的列必须定义为非空.</p>
-<p>WHERE子句使用的索引和ORDER BY子句中所使用的索引不能并列.</p></li>
+<p><span class="math display">\[
+\frac {\partial y_i}{\partial x_i} = \frac {e^{x_i}\left (\sum_{k=1}^n
+e^{x_k}\right) - e^{x_i} e^{x_i}}{\left (\sum_{k=1}^n e^{x_k}\right)^2}
+= y_i (1-y_i)
+\]</span></p>
+<ol start="2" type="1">
+<li>当 <span class="math inline">\(i \neq j\)</span> 时：</li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效: (索引不被使用)</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_CODE <span class="keyword">FROM</span> DEPT <span class="keyword">ORDER</span> <span class="keyword">BY</span> DEPT_TYPE</span><br><span class="line">高效: (使用索引)</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_CODE <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> DEPT_TYPE <span class="operator">&gt;</span> <span class="number">0</span></span><br></pre></td></tr></table></figure>
-<ol start="29" type="1">
-<li><p>避免改变索引列的类型</p>
-<p>当比较不同数据类型的数据时,
-ORACLE自动对列进行简单的类型转换.</p></li>
+<p><span class="math display">\[
+\frac {\partial y_i}{\partial x_j} = -\frac {e^{x_j} e^{x_i}}{\left
+(\sum_{k=1}^n e^{x_k}\right)^2} = -y_i y_j
+\]</span></p></li>
+<li><p>由于求导的链式法则，因此将SoftMax的导数乘以交叉熵的导数，就能得到模型输出的梯度。综上，对于模型的输出，交叉熵的梯度仅在目标标签上有效（≠0），所以最后的梯度计算为，</p></li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 假设 EMPNO是一个数值类型的索引列.</span><br><span class="line"># 低效</span><br><span class="line"><span class="keyword">SELECT</span> … <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> EMPNO <span class="operator">=</span> <span class="string">&#x27;123&#x27;</span></span><br><span class="line"># 高效</span><br><span class="line"><span class="keyword">SELECT</span> … <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> EMPNO <span class="operator">=</span> <span class="number">123</span></span><br></pre></td></tr></table></figure>
-<ol start="30" type="1">
-<li><p>需要当心的WHERE子句</p>
-<p>某些SELECT 语句中的WHERE子句不使用索引. 这里有一些例子.</p>
+<p><span class="math display">\[-\frac{1}{\hat{y}_c}\times
+y_c(1-y_c)=y_c-1\]</span></p>
+<h3 id="autograd">autograd</h3>
+<p>在PyTorch内部有一套完善的求导机制。因此，仅需按照其规则进行书写即可实现前向传播、反向传播（链式求导）。如下所示，在forward填写loss的计算，backward中填写grad的计算即可</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">CrossEntropyLoss</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">        <span class="comment"># outputs：模型的输出</span></span><br><span class="line">        <span class="comment"># targets: 真实标签</span></span><br><span class="line">        ...</span><br><span class="line">        ctx.save_for_backward(...) <span class="comment"># 需要在反向传播中使用的变量</span></span><br><span class="line">        <span class="keyword">return</span> loss</span><br><span class="line">    </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">        <span class="comment"># grad_output: 反向传播的梯度</span></span><br><span class="line">        ... = ctx.saved_tensors</span><br><span class="line">        ...</span><br><span class="line">	    <span class="keyword">return</span> new_grad_output</span><br></pre></td></tr></table></figure>
+<h4 id="forward">Forward</h4>
+<p>这里基本就是把公式照抄过来，但需要额外把梯度计算时用到的变量存一下。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  </span><br><span class="line"></span><br><span class="line">    loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) </span><br><span class="line">    ctx.save_for_backward(outputs, targets)</span><br><span class="line">    <span class="keyword">return</span> loss.mean()</span><br></pre></td></tr></table></figure>
+<h4 id="backward">Backward</h4>
+<p>这里的<code>grad_output</code>
+表示梯度计算的输出。它是一个与前向传播函数的输出形状相同的张量。在反向传播过程中，我们将计算当前函数的导数，乘以
+<code>grad_output</code>
+作为输入。这个值代表了后续节点对当前节点的梯度贡献。</p>
+<p>通过将 <code>grad_output</code>
+乘以导数（也称为雅可比向量积），可以有效地传递梯度信息到较早的节点，从而实现自动微分。</p>
+<p>注意，<code>grad_output</code>
+的形状必须与函数的输出形状一致，否则会引发错误。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">    outputs, targets = ctx.saved_tensors</span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    </span><br><span class="line">    grad_y_pred = outputs.clone()</span><br><span class="line">    grad_y_pred[<span class="built_in">range</span>(bs), targets] -= <span class="number">1</span>  <span class="comment"># 计算对应类别的梯度</span></span><br><span class="line">    grad_y_pred /= bs <span class="comment"># 取均值</span></span><br><span class="line">    </span><br><span class="line">    grad_y_pred *= grad_output.item()  <span class="comment"># 乘以关于loss的梯度</span></span><br><span class="line">    </span><br><span class="line">    <span class="keyword">return</span> grad_y_pred, <span class="literal">None</span></span><br></pre></td></tr></table></figure>
+<h3 id="完整版">完整版</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyCrossEntropyLossFunction</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, outputs, targets</span>):</span><br><span class="line">        bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">        outputs = torch.softmax(outputs, dim=<span class="number">1</span>)  </span><br><span class="line"></span><br><span class="line">        loss = -torch.log(outputs[<span class="built_in">range</span>(bs), targets]) </span><br><span class="line">        ctx.save_for_backward(outputs, targets)</span><br><span class="line">        <span class="keyword">return</span> loss.mean()</span><br><span class="line">    </span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line">        outputs, targets = ctx.saved_tensors</span><br><span class="line">        bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">        </span><br><span class="line">        grad_y_pred = outputs.clone()</span><br><span class="line">        grad_y_pred[<span class="built_in">range</span>(bs), targets] -= <span class="number">1</span>  <span class="comment"># 只需计算对应标签的梯度</span></span><br><span class="line">        </span><br><span class="line">        grad_y_pred /= bs <span class="comment"># 取均值</span></span><br><span class="line">        </span><br><span class="line">        grad_y_pred *= grad_output.item()  <span class="comment"># 乘以关于loss的梯度</span></span><br><span class="line">        </span><br><span class="line">        <span class="keyword">return</span> grad_y_pred, <span class="literal">None</span></span><br><span class="line">  </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">MyCrossEntropyLoss</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="keyword">return</span> MyCrossEntropyLossFunction.apply(outputs, targets)</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]], requires_grad=<span class="literal">True</span>)</span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])</span><br><span class="line">loss = MyCrossEntropyLoss(outputs, targets)</span><br><span class="line"><span class="comment"># tensor(0.9398, grad_fn=&lt;MyCrossEntropyLossFunctionBackward&gt;)</span></span><br></pre></td></tr></table></figure>
+<p>结合简单的双层MLP</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">import</span> copy</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MLP</span>(torch.nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="built_in">super</span>().__init__()</span><br><span class="line">        <span class="variable language_">self</span>.l1 = torch.nn.Linear(<span class="number">5</span>, <span class="number">5</span>, bias=<span class="literal">False</span>)</span><br><span class="line">        <span class="variable language_">self</span>.l2 = torch.nn.Linear(<span class="number">5</span>, <span class="number">3</span>, bias=<span class="literal">False</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, x</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.l2(<span class="variable language_">self</span>.l1(x))</span><br><span class="line">    </span><br><span class="line">input_ = torch.rand((<span class="number">5</span>, <span class="number">5</span>))</span><br><span class="line">labels = torch.randint(<span class="number">0</span>, <span class="number">3</span>, size=(<span class="number">5</span>,))</span><br><span class="line">model1 = MLP()</span><br><span class="line">model2 = copy.deepcopy(model1)</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">10</span>, <span class="string">&quot;Before Train&quot;</span>, <span class="string">&quot;=&quot;</span>*<span class="number">10</span>)</span><br><span class="line"><span class="built_in">print</span>(model1.l1.weight)</span><br><span class="line"><span class="built_in">print</span>(model2.l1.weight)</span><br><span class="line"></span><br><span class="line">criterion1 = torch.nn.CrossEntropyLoss()</span><br><span class="line">criterion2 = MyCrossEntropyLoss</span><br><span class="line"></span><br><span class="line">optim1 = torch.optim.Adam(model1.parameters(), lr=<span class="number">1e-2</span>)</span><br><span class="line">optim2 = torch.optim.Adam(model2.parameters(), lr=<span class="number">1e-2</span>)</span><br><span class="line"></span><br><span class="line">outputs1 = model1(copy.deepcopy(input_))</span><br><span class="line">loss1 = criterion1(outputs1, labels)</span><br><span class="line">optim1.zero_grad()</span><br><span class="line">loss1.backward()</span><br><span class="line">optim1.step()</span><br><span class="line"></span><br><span class="line">outputs2 = model2(copy.deepcopy(input_))</span><br><span class="line">loss2 = criterion2(outputs2, labels)</span><br><span class="line">optim2.zero_grad()</span><br><span class="line">loss2.backward()</span><br><span class="line">optim2.step()</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">&quot;=&quot;</span>*<span class="number">10</span>, <span class="string">&quot;After Train&quot;</span>, <span class="string">&quot;=&quot;</span>*<span class="number">10</span>)</span><br><span class="line"><span class="built_in">print</span>(model1.l1.weight)</span><br><span class="line"><span class="built_in">print</span>(model2.l1.weight)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h2 id="step-3-分布式">Step 3: 分布式</h2>
+<p>模型训练速度提升+单卡放不下大模型，由于这两个需求，所以分布式计算中需要重写损失函数的forward和backward，以Megatron-LM的训练代码为例，如下所示。接下来将逐步拆解实现过程（忽略label_smoothing，将其看作0.0）。</p>
+<p><a
+href="https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/cross_entropy.py">cross_entropy.py</a></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> megatron.core.parallel_state <span class="keyword">import</span> (</span><br><span class="line">    get_tensor_model_parallel_group,</span><br><span class="line">    get_tensor_model_parallel_rank,</span><br><span class="line">    get_tensor_model_parallel_world_size,</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> .utils <span class="keyword">import</span> VocabUtility</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">_VocabParallelCrossEntropy</span>(torch.autograd.Function):</span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">ctx, vocab_parallel_logits, target, label_smoothing=<span class="number">0.0</span></span>):</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Maximum value along vocab dimension across all GPUs.</span></span><br><span class="line">        logits_max = torch.<span class="built_in">max</span>(vocab_parallel_logits, dim=-<span class="number">1</span>)[<span class="number">0</span>]</span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group()</span><br><span class="line">        )</span><br><span class="line">        <span class="comment"># Subtract the maximum value.</span></span><br><span class="line">        vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Get the partition&#x27;s vocab indecies</span></span><br><span class="line">        get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size</span><br><span class="line">        partition_vocab_size = vocab_parallel_logits.size()[-<span class="number">1</span>]</span><br><span class="line">        rank = get_tensor_model_parallel_rank()</span><br><span class="line">        world_size = get_tensor_model_parallel_world_size()</span><br><span class="line">        vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Create a mask of valid vocab ids (1 means it needs to be masked).</span></span><br><span class="line">        target_mask = (target &lt; vocab_start_index) | (target &gt;= vocab_end_index)</span><br><span class="line">        masked_target = target.clone() - vocab_start_index</span><br><span class="line">        masked_target[target_mask] = <span class="number">0</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Get predicted-logits = logits[target].</span></span><br><span class="line">        <span class="comment"># For Simplicity, we convert logits to a 2-D tensor with size</span></span><br><span class="line">        <span class="comment"># [*, partition-vocab-size] and target to a 1-D tensor of size [*].</span></span><br><span class="line">        logits_2d = vocab_parallel_logits.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line">        masked_target_1d = masked_target.view(-<span class="number">1</span>)</span><br><span class="line">        arange_1d = torch.arange(start=<span class="number">0</span>, end=logits_2d.size()[<span class="number">0</span>], device=logits_2d.device)</span><br><span class="line">        predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]</span><br><span class="line">        predicted_logits_1d = predicted_logits_1d.clone().contiguous()</span><br><span class="line">        predicted_logits = predicted_logits_1d.view_as(target)</span><br><span class="line">        predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line">        <span class="comment"># All reduce is needed to get the chunks from other GPUs.</span></span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            predicted_logits,</span><br><span class="line">            op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">            group=get_tensor_model_parallel_group(),</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Sum of exponential of logits along vocab dimension across all GPUs.</span></span><br><span class="line">        exp_logits = vocab_parallel_logits</span><br><span class="line">        torch.exp(vocab_parallel_logits, out=exp_logits)</span><br><span class="line">        sum_exp_logits = exp_logits.<span class="built_in">sum</span>(dim=-<span class="number">1</span>)</span><br><span class="line">        torch.distributed.all_reduce(</span><br><span class="line">            sum_exp_logits,</span><br><span class="line">            op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">            group=get_tensor_model_parallel_group(),</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Loss = log(sum(exp(logits))) - predicted-logit.</span></span><br><span class="line">        loss = torch.log(sum_exp_logits) - predicted_logits</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Normalize and optionally smooth logits</span></span><br><span class="line">        exp_logits.div_(sum_exp_logits.unsqueeze(dim=-<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line">        vocab_size = exp_logits.size(-<span class="number">1</span>)</span><br><span class="line">        <span class="keyword">if</span> label_smoothing &gt; <span class="number">0</span>:</span><br><span class="line">            <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            We&#x27;d like to assign 1 / (K - 1) probability mass to every index that is not the ground truth.</span></span><br><span class="line"><span class="string">            = (1 - alpha) * y_gt + alpha * mean(y_&#123;i for i != gt&#125;)</span></span><br><span class="line"><span class="string">            = (1 - alpha) * y_gt + (alpha / (K - 1)) * \sum_&#123;i != gt&#125; y_i</span></span><br><span class="line"><span class="string">            = ((K - 1) * (1 - alpha) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_&#123;i != gt&#125; y_i</span></span><br><span class="line"><span class="string">            = (K * (1 - alpha) - 1) / (K - 1)) * y_gt  + (alpha / (K - 1)) * \sum_&#123;i&#125; y_i</span></span><br><span class="line"><span class="string">            = (1 - (alpha * K) / (K - 1)) * y_gt + ( (alpha * K) / (K - 1) ) * \sum_&#123;i&#125; y_i / K</span></span><br><span class="line"><span class="string">            From: https://github.com/NVIDIA/NeMo/blob/main/nemo/collections/common/losses/smoothed_cross_entropy.py</span></span><br><span class="line"><span class="string">            &quot;&quot;&quot;</span></span><br><span class="line">            <span class="keyword">assert</span> <span class="number">1.0</span> &gt; label_smoothing &gt; <span class="number">0.0</span></span><br><span class="line">            smoothing = label_smoothing * vocab_size / (vocab_size - <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">            <span class="comment"># Exp logits at this point are normalized probabilities. So we can just take the log to get log-probs.</span></span><br><span class="line">            log_probs = torch.log(exp_logits)</span><br><span class="line">            mean_log_probs = log_probs.mean(dim=-<span class="number">1</span>)</span><br><span class="line">            loss = (<span class="number">1.0</span> - smoothing) * loss - smoothing * mean_log_probs</span><br><span class="line"></span><br><span class="line">        ctx.label_smoothing, ctx.vocab_size = label_smoothing, vocab_size</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Store softmax, target-mask and masked-target for backward pass.</span></span><br><span class="line">        ctx.save_for_backward(exp_logits, target_mask, masked_target_1d)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> loss</span><br><span class="line"></span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">backward</span>(<span class="params">ctx, grad_output</span>):</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Retreive tensors from the forward path.</span></span><br><span class="line">        softmax, target_mask, masked_target_1d = ctx.saved_tensors</span><br><span class="line">        label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size</span><br><span class="line"></span><br><span class="line">        <span class="comment"># All the inputs have softmax as thier gradient.</span></span><br><span class="line">        grad_input = softmax</span><br><span class="line">        <span class="comment"># For simplicity, work with the 2D gradient.</span></span><br><span class="line">        partition_vocab_size = softmax.size()[-<span class="number">1</span>]</span><br><span class="line">        grad_2d = grad_input.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Add the gradient from matching classes.</span></span><br><span class="line">        arange_1d = torch.arange(start=<span class="number">0</span>, end=grad_2d.size()[<span class="number">0</span>], device=grad_2d.device)</span><br><span class="line"></span><br><span class="line">        softmax_update = <span class="number">1.0</span> - target_mask.view(-<span class="number">1</span>).<span class="built_in">float</span>()</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> label_smoothing &gt; <span class="number">0</span>:</span><br><span class="line">            smoothing = label_smoothing * vocab_size / (vocab_size - <span class="number">1</span>)</span><br><span class="line">            grad_2d[arange_1d, masked_target_1d] -= (<span class="number">1.0</span> - smoothing) * softmax_update</span><br><span class="line">            average_grad = <span class="number">1</span> / vocab_size</span><br><span class="line">            grad_2d[arange_1d, :] -= smoothing * average_grad</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            grad_2d[arange_1d, masked_target_1d] -= softmax_update</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Finally elementwise multiplication with the output gradients.</span></span><br><span class="line">        grad_input.mul_(grad_output.unsqueeze(dim=-<span class="number">1</span>))</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> grad_input, <span class="literal">None</span>, <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">vocab_parallel_cross_entropy</span>(<span class="params">vocab_parallel_logits, target, label_smoothing=<span class="number">0.0</span></span>):</span><br><span class="line">    <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">    Performs cross entropy loss when logits are split across tensor parallel ranks</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Arguments:</span></span><br><span class="line"><span class="string">        vocab_parallel_logits: logits split across tensor parallel ranks</span></span><br><span class="line"><span class="string">                               dimension is [sequence_length, batch_size, hidden_size]</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        target: correct vocab ids of dimseion [sequence_length, micro_batch_size]</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">        lobal_smoothing: smoothing factor, must be in range [0.0, 1.0)</span></span><br><span class="line"><span class="string">                         default is no smoothing (=0.0)</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    <span class="keyword">return</span> _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, label_smoothing)</span><br></pre></td></tr></table></figure>
+<h3 id="为什么">为什么</h3>
+<p>本文不介绍分布式训练的其他原理，仅关注模型最后的输出，以及损失函数的计算。</p>
+<p>前文已经详细介绍了交叉熵的实现，那么为什么到分布式训练的时候，就不能直接复用呢？主要原因在于模型的输出被拆散了。回顾模型的输出，其shape为[<code>bs</code>,
+<code>C</code>]，<code>bs</code>表示每次训练的数据量<code>batch size</code>，<code>C</code>表示任务的总的分类数量，在LLM中主要是词表大小<code>vocab_size</code>（下文用此代表类别数量）。</p>
+<p>对于模型的输出而言，分布式（假设有<code>word_size</code>张GPU）主要分两种情况
+1.
+拆分<code>bs</code>，每张卡上有<code>bs</code>数据量，那么整合模型的最后输出应为[<code>bs</code>
+* <code>word_size</code>, <code>vocab_size</code>] 2.
+拆分<code>vocab_size</code>，单个模型的输出为[<code>bs</code>,
+<code>vocab_size</code> //
+<code>word_size</code>]。这种情况需要保证<code>vocab_size</code>能够整除<code>word_size</code></p>
+<p>因此，在进行loss计算前，需要先同步所有的模型输出，然后再进行loss计算。这里的同步，就是从所有GPU中获取模型的输出。但考虑到通信开销问题，有些操作是可以进行优化的，即无需通信全部内容再计算。重新思考损失的计算过程可以分为如下几步：</p>
 <ol type="1">
-<li><code>!=</code> 将不使用索引. 记住, 索引只能告诉你什么存在于表中,
-而不能告诉你什么不存在于表中.</li>
-<li><code>¦ ¦</code>是字符连接函数. 就象其他函数那样, 停用了索引.</li>
-<li><code>+</code>是数学函数. 就象其他数学函数那样, 停用了索引.</li>
-<li>相同的索引列不能互相比较,这将会启用全表扫描.</li>
-</ol></li>
-<li><ol type="a">
-<li><p>如果检索数据量超过30%的表中记录数.使用索引将没有显著的效率提高.</p></li>
-<li><p>在特定情况下, 使用索引也许会比全表扫描慢,
-但这是同一个数量级上的区别.
-而通常情况下,使用索引比全表扫描要块几倍乃至几千倍!</p></li>
+<li>SoftMax计算：<span class="math inline">\(\exp(x) /
+\sum(\exp(x)\)</span></li>
+<li>log的计算： <span class="math inline">\(-\log[\exp(x) /
+\sum(\exp(x)]\)</span></li>
+<li>索引：根据targets索引矩阵对应位置 <span class="math inline">\(x[...,
+...]\)</span></li>
+</ol>
+<p>其中，前两者跟最后一个步骤是可以独立运行的。而前两者的计算是耦合的，为节约通信量，对log计算进行变化</p>
+<ol type="1">
+<li><span class="math inline">\(-\log[\exp(x) /
+\sum(\exp(x)]\)</span></li>
+<li><span class="math inline">\(-\log[\exp(x)] +
+\log[\sum\exp(x)]\)</span></li>
+<li><span class="math inline">\(\log[\sum\exp(x)] - x\)</span></li>
+</ol>
+<p>因此，可以由原始的<code>func1</code>推导至分布式的<code>func2</code>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func1</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># 原始版本</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line">    <span class="comment"># Step 0:</span></span><br><span class="line">    predicted_logits = outputs[<span class="built_in">range</span>(bs), targets]</span><br><span class="line"></span><br><span class="line">    predicted_logits = torch.softmax(predicted_logits, dim=<span class="number">1</span>)</span><br><span class="line">    loss = -torch.log(predicted_logits)</span><br><span class="line">    <span class="built_in">print</span>(loss)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func2</span>(<span class="params">outputs, targets</span>):</span><br><span class="line">    <span class="comment"># 分布式的简化版本</span></span><br><span class="line">    bs = outputs.size(<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">    min_c, max_c = <span class="number">0</span>, <span class="number">3</span> <span class="comment"># 预先设定的最小和最大值</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 只选择在该设备上的数值</span></span><br><span class="line">    target_mask = (targets &lt; min_c) | (targets &gt;= max_c)</span><br><span class="line">    <span class="comment"># 由于在分布式中min_c不一定为0，所以需要做一个归一化</span></span><br><span class="line">    masked_target = targets.clone() - min_c</span><br><span class="line">    masked_target[target_mask] = min_c</span><br><span class="line"></span><br><span class="line">    <span class="comment"># arange_1d, masked_target 分别对应 上文的range(bs), targets</span></span><br><span class="line">    arange_1d = torch.arange(start=<span class="number">0</span>, end=bs)</span><br><span class="line">    predicted_logits = outputs[arange_1d, masked_target]</span><br><span class="line">    </span><br><span class="line">    predicted_logits = predicted_logits.clone().contiguous()</span><br><span class="line">    <span class="comment"># 对于非该设备上计算的值，置为0。则不进行计算</span></span><br><span class="line">    predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line">    <span class="comment"># 先算第二项</span></span><br><span class="line">    predicted_logits = torch.<span class="built_in">sum</span>(predicted_logits, dim=-<span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">    loss = torch.log(torch.exp(outputs).<span class="built_in">sum</span>(dim=-<span class="number">1</span>)) - predicted_logits</span><br><span class="line">    <span class="built_in">print</span>(loss)</span><br><span class="line"></span><br><span class="line">outputs = torch.tensor([[<span class="number">0.5</span>, <span class="number">0.2</span>, <span class="number">0.3</span>]])</span><br><span class="line">targets = torch.tensor([<span class="number">0</span>])    </span><br><span class="line"></span><br><span class="line">func1(outputs, targets)</span><br><span class="line">func2(outputs, targets)</span><br></pre></td></tr></table></figure></p>
+<h3 id="forward-1">Forward</h3>
+<h4 id="预处理">预处理</h4>
+<p>一般而言，为了数值稳定性，SoftMax计算前会减去最大值，即</p>
+<p><span class="math display">\[\text {softmax}(x_i) = \frac
+{e^{x_i}}{\sum_{j=1}^n e^{x_j}} = \frac {e^{x_i - \max(x)}}{\sum_{j=1}^n
+e^{x_j - \max(x)}}\]</span></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 每张卡上的模型输出为 vocab_parallel_logits，shape：[bs, seq_len, vocab_size // word_size]</span></span><br><span class="line"><span class="comment"># SoftMax的前置计算，可以无需通信全部参数，分两步进行</span></span><br><span class="line"><span class="comment"># 1. 计算每张卡的最大值</span></span><br><span class="line"><span class="comment"># 2. 再计算所有卡的最大值</span></span><br><span class="line"><span class="comment"># 3. 最后对每卡张减去最大值</span></span><br><span class="line"><span class="comment"># 这样就能避免通信所有的参数</span></span><br><span class="line">logits_max = torch.<span class="built_in">max</span>(vocab_parallel_logits, dim=-<span class="number">1</span>)[<span class="number">0</span>]</span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group()</span><br><span class="line">)</span><br><span class="line"><span class="comment"># Subtract the maximum value.</span></span><br><span class="line">vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
+<p>这里的<code>seq_len</code>是LLM中句子的长度，句子的每个位置（token）会计算下一个token的概率（在词表中选），因此，可以理解为每个位置都是一个分类任务，因此<code>vocab_size</code>就是分类的数量。即，可以转换为[<code>bs</code>*<code>seq_len</code>,
+<code>vocab_size</code> // <code>word_size</code>]的形式</p>
+<p>由于单卡不一定存在所有词表的值，因此需要在这张卡上将非该卡输出位置的值置为0。
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Get the partition&#x27;s vocab indecies</span></span><br><span class="line">get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size</span><br><span class="line">partition_vocab_size = vocab_parallel_logits.size()[-<span class="number">1</span>]</span><br><span class="line">rank = get_tensor_model_parallel_rank()</span><br><span class="line">world_size = get_tensor_model_parallel_world_size()</span><br><span class="line">vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Create a mask of valid vocab ids (1 means it needs to be masked).</span></span><br><span class="line">target_mask = (target &lt; vocab_start_index) | (target &gt;= vocab_end_index)</span><br><span class="line">masked_target = target.clone() - vocab_start_index</span><br><span class="line">masked_target[target_mask] = <span class="number">0</span></span><br></pre></td></tr></table></figure></p>
+<h4 id="根据-targets-选择模型真实输出的logits">根据 targets
+选择模型真实输出的Logits</h4>
+<p>将模型的输出和标签 转换为[<code>bs</code>*<code>seq_len</code>,
+<code>vocab_size</code> // <code>word_size</code>]的形式，以便于计算
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">logits_2d = vocab_parallel_logits.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line">masked_target_1d = masked_target.view(-<span class="number">1</span>) </span><br><span class="line"></span><br><span class="line"><span class="comment"># 选择真实标签的logits，相当于原来的outputs[range(bs), targets]</span></span><br><span class="line"><span class="comment"># 相当于把此步骤置前，而对于后续计算无影响，并减少了后续的通信时间</span></span><br><span class="line">arange_1d = torch.arange(start=<span class="number">0</span>, end=logits_2d.size()[<span class="number">0</span>], device=logits_2d.device)</span><br><span class="line">predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]</span><br><span class="line">predicted_logits_1d = predicted_logits_1d.clone().contiguous()</span><br></pre></td></tr></table></figure></p>
+<h4 id="计算loss">计算loss</h4>
+<p>将logits转换为原来的shape，再求和，先得到了最后的第一项，之后再减去log
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">predicted_logits = predicted_logits_1d.view_as(target)</span><br><span class="line">predicted_logits[target_mask] = <span class="number">0.0</span></span><br><span class="line"><span class="comment"># All reduce is needed to get the chunks from other GPUs.</span></span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    predicted_logits,</span><br><span class="line">    op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">    group=get_tensor_model_parallel_group(),</span><br><span class="line">)</span><br></pre></td></tr></table></figure></p>
+<p>计算SoftMax <span class="math display">\[\text {softmax}(x_i) = \frac
+{e^{x_i}}{\sum_{j=1}^n e^{x_j}}\]</span></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">exp_logits = vocab_parallel_logits</span><br><span class="line"><span class="comment"># 计算每张卡的exp，作为分子</span></span><br><span class="line">torch.exp(vocab_parallel_logits, out=exp_logits)</span><br><span class="line"><span class="comment"># 再计算这张卡上的exp之和</span></span><br><span class="line">sum_exp_logits = exp_logits.<span class="built_in">sum</span>(dim=-<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 再计算所有卡的exp之和，作为分母</span></span><br><span class="line">torch.distributed.all_reduce(</span><br><span class="line">    sum_exp_logits,</span><br><span class="line">    op=torch.distributed.ReduceOp.SUM,</span><br><span class="line">    group=get_tensor_model_parallel_group(),</span><br><span class="line">)</span><br></pre></td></tr></table></figure>
+<p>计算loss <figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 第一项是log(sum(exp(logits)))，第二项是最后的logits</span></span><br><span class="line">loss = torch.log(sum_exp_logits) - predicted_logits</span><br><span class="line"></span><br><span class="line"><span class="comment"># 为了Backward，需要保存一些变量</span></span><br><span class="line">exp_logits.div_(sum_exp_logits.unsqueeze(dim=-<span class="number">1</span>))</span><br></pre></td></tr></table></figure></p>
+<h3 id="backward-1">Backward</h3>
+<h4 id="预处理-1">预处理</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">softmax, target_mask, masked_target_1d = ctx.saved_tensors</span><br><span class="line">label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据SoftMax计算的结果，忽略无需计算梯度的位置。target_mask中的元素不是0就是1</span></span><br><span class="line">grad_input = softmax</span><br><span class="line"><span class="comment"># For simplicity, work with the 2D gradient.</span></span><br><span class="line">partition_vocab_size = softmax.size()[-<span class="number">1</span>]</span><br><span class="line">grad_2d = grad_input.view(-<span class="number">1</span>, partition_vocab_size)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Add the gradient from matching classes.</span></span><br><span class="line">arange_1d = torch.arange(start=<span class="number">0</span>, end=grad_2d.size()[<span class="number">0</span>], device=grad_2d.device)</span><br><span class="line"></span><br><span class="line">softmax_update = <span class="number">1.0</span> - target_mask.view(-<span class="number">1</span>).<span class="built_in">float</span>()</span><br></pre></td></tr></table></figure>
+<h4 id="梯度计算">梯度计算</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 正常计算位置，就-1，否则将不减（即不做计算）</span></span><br><span class="line">grad_2d[arange_1d, masked_target_1d] -= softmax_update</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最后乘上关于loss的梯度</span></span><br><span class="line">grad_input.mul_(grad_output.unsqueeze(dim=-<span class="number">1</span>))</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Algorithm</category>
+      </categories>
+      <tags>
+        <tag>PyTorch</tag>
+        <tag>数学</tag>
+        <tag>交叉熵</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>torch中的scheduler</title>
+    <url>/hexo_blog/2021/02/03/PyTorch/torch-scheduler%E8%AE%B0%E5%BD%95/</url>
+    <content><![CDATA[<p>PyTorch学习率调整过程中版本问题引发的不同结果解析</p>
+<span id="more"></span>
+<p>代码</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line">optimizer1 = torch.optim.SGD([torch.randn(<span class="number">1</span>, requires_grad=<span class="literal">True</span>)], lr=<span class="number">1e-3</span>)</span><br><span class="line">exp_lr_scheduler1 = torch.optim.lr_scheduler.MultiStepLR(optimizer1,</span><br><span class="line">                                             milestones=[<span class="number">5</span>, <span class="number">10</span>], gamma=<span class="number">0.1</span>)</span><br><span class="line"></span><br><span class="line">optimizer2 = torch.optim.SGD([torch.randn(<span class="number">1</span>, requires_grad=<span class="literal">True</span>)], lr=<span class="number">1e-3</span>)</span><br><span class="line">exp_lr_scheduler2 = torch.optim.lr_scheduler.MultiStepLR(optimizer2,</span><br><span class="line">                                             milestones=[<span class="number">5</span>, <span class="number">10</span>], gamma=<span class="number">0.1</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">15</span>):</span><br><span class="line">    exp_lr_scheduler1.step()</span><br><span class="line">    exp_lr_scheduler2.step(epoch)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;Epoch &#123;&#125;, lr1 &#123;&#125;, lr2 &#123;&#125;&#x27;</span>.<span class="built_in">format</span>(epoch,</span><br><span class="line">        optimizer1.param_groups[<span class="number">0</span>][<span class="string">&#x27;lr&#x27;</span>],</span><br><span class="line">        optimizer2.param_groups[<span class="number">0</span>][<span class="string">&#x27;lr&#x27;</span>]))</span><br></pre></td></tr></table></figure>
+<p>当torch版本为1.2.0时，输出如下：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Epoch <span class="number">1</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">2</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">3</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">4</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">0.001</span></span><br><span class="line">Epoch <span class="number">5</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">6</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">7</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">8</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">9</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">0.0001</span></span><br><span class="line">Epoch <span class="number">10</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">11</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">12</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">13</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">14</span>, lr1 <span class="number">1.0000000000000003e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br></pre></td></tr></table></figure>
+<p>当torch版本为1.4.0时，输出如下：</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Epoch <span class="number">1</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">2</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">3</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">4</span>, lr1 <span class="number">0.001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">5</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">6</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">7</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">8</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">9</span>, lr1 <span class="number">0.0001</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">10</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">11</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">12</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">13</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br><span class="line">Epoch <span class="number">14</span>, lr1 <span class="number">1e-05</span>, lr2 <span class="number">1.0000000000000003e-05</span></span><br></pre></td></tr></table></figure>
+<p>小结：<code>scheduler.step()</code>中的
+<code>epoch</code>参数由于版本问题会带来不同的作用效果。1.2.0版本是判断epoch是否在某个区间内，而1.4.0版本是会直接调整学习到达最后一个区间。</p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>PyTorch</tag>
+        <tag>scheduler</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Birthday in Pi：查找日期在Pi中的位置</title>
+    <url>/hexo_blog/2021/03/18/Python/Birthday_in_Pi/</url>
+    <content><![CDATA[<p>想知道自己的生日在圆周率π的第几位出现吗？本文介绍了从思路到实现所遇到的问题，详细分析了如何使用迭代器优化性能，并改用Python语言实现了类似“
+锁”内存等操作。最终，将函数打包成了exe程序，在公众号内提供查询服务，还提供了一个不断更新的生日与位置的查表。</p>
+<span id="more"></span>
+<p>完整的项目代码均开源: <a
+href="https://github.com/wnma3mz/birthday_in_pi">https://github.com/wnma3mz/birthday_in_pi</a></p>
+<h2 id="想法">想法</h2>
+<p>于我而言，很多项目的出发点都是源于浏览东西时所想，觉得有意思就去实现。比如这次是在即刻上看到北大数院公众号，可以回复生日（如20000101）就可以知道这串数字在Pi的第多少位出现。更详细的例子：已知3.1415926...，那么1415就是出现在第1位（从1开始计数，小数点后开始计数）。但该公众号仅在Pi的2亿位数中进行查找，所以有即友回复，搜不到结果。故想到可以试试更长的数据集是否能够搜到。</p>
+<p><strong>背景知识</strong></p>
+<p>理论上，Pi里面可以查找所有人的电话号码、生日、银行卡密码等等。</p>
+<h2 id="行动">行动</h2>
+<h3 id="数据">数据</h3>
+<p>巧妇难为无米之炊，这里最重要的就是Pi这个无理数数据。</p>
+<ol type="1">
+<li>是否有现成的数据文本-&gt;没搜到，搜到的位数都很小，不足以满足需求</li>
+<li>计算Pi（生成超过2亿数的文本）
+<ol type="1">
+<li>自己手写</li>
+<li>找程序。对于计算Pi的项目，本身就是一个科学问题，故一定有现有程序可以辅助完成。所以找到了<a
+href="http://www.numberworld.org/y-cruncher/">y-cruncher</a>项目。具体是否为当前最快的生成程序，未深究，不过实际测试，效率已经可以满足需求。（用i5-1035G1的CPU，生成25亿数，8个线程计算用了10分钟不到，文本大小2.4G）</li>
 </ol></li>
-<li><p>避免使用耗费资源的操作</p>
-<p>带有DISTINCT,UNION,MINUS,INTERSECT,ORDER
-BY的SQL语句会启动SQL引擎执行耗费资源的排序(SORT)功能.
-DISTINCT需要一次排序操作, 而其他的至少需要执行两次排序. 通常, 带有UNION,
-MINUS , INTERSECT的SQL语句都可以用其他方式重写.
-如果你的数据库的SORT_AREA_SIZE调配得好, 使用UNION , MINUS,
-INTERSECT也是可以考虑的, 毕竟它们的可读性很强。</p></li>
-<li><p>优化GROUP BY</p>
-<p>提高GROUP BY 语句的效率, 可以通过将不需要的记录在GROUP BY
-之前过滤掉.下面两个查询返回相同结果但第二个明显就快了许多.</p></li>
 </ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> JOB, <span class="built_in">AVG</span>(SAL)</span><br><span class="line"><span class="keyword">FROM</span> EMP</span><br><span class="line"><span class="keyword">GROUP</span> <span class="keyword">by</span> JOB</span><br><span class="line"><span class="keyword">HAVING</span> JOB <span class="operator">=</span> <span class="string">&#x27;PRESIDENT&#x27;</span></span><br><span class="line"><span class="keyword">OR</span> JOB <span class="operator">=</span> <span class="string">&#x27;MANAGER&#x27;</span></span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> JOB, <span class="built_in">AVG</span>(SAL)</span><br><span class="line"><span class="keyword">FROM</span> EMP</span><br><span class="line"><span class="keyword">WHERE</span> JOB <span class="operator">=</span> <span class="string">&#x27;PRESIDENT&#x27;</span></span><br><span class="line"><span class="keyword">OR</span> JOB <span class="operator">=</span> <span class="string">&#x27;MANAGER&#x27;</span></span><br><span class="line"><span class="keyword">GROUP</span> <span class="keyword">by</span> JOB</span><br></pre></td></tr></table></figure>
-<pre><code>​</code></pre>
+<h3 id="编码">编码</h3>
+<p>该问题本身简单来看就是一个字符串查找问题，所以用Python的话，<code>find</code>函数可以直接完成这项任务。但一旦问题变"大"（数据集变大、机器变多......），就是一个"难"题了。</p>
+<p>当然，先不管如何，试试最粗暴的解法看看性能如何。根据测试结果，有两个问题，时间超过5s,
+内存占用2G（文本大小2G）。就算时间可以接受，那么这个空间占用也很难接受。试想，每查询一次都需要占用2G内存，查询完再释放。所以直接想到的优化点有以下几个</p>
+<ol type="1">
+<li>"锁"内存，即每次查询完不释放这个内存，这样IO时间就可以大大减小</li>
+<li>"做表"，查询的生日数字是有限的，无非就是(19700101-20101231有限数据)，就算100年，也只是36500个数据（实际会略高，因为有闰年）。查表的效率就非常快了</li>
+<li>从根本上优化这个程序</li>
+</ol>
+<h3 id="优化">优化</h3>
+<p>首先排除2，因为最终还是想查找任意数字，2不便于后续扩展，而且根据现有的性能，做表也需要很长时间。</p>
+<p>其次对于1，如果考虑对外开放，服务器占用2G内存空间，实在是太"奢侈"了</p>
+<p>所以最后就只能考虑3。</p>
+<h4 id="迭代器">迭代器</h4>
+<p>Python有个关键词是
+<code>yield</code>，它可以实现迭代器这个功能。说人话就是，它不是一次性把数据加载出来，而且每次只加载一部分。下面是一个简单的例子</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">yield_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> lst:</span><br><span class="line">        <span class="keyword">yield</span> i</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">for_return_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> lst:</span><br><span class="line">        <span class="keyword">return</span> i</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">return_func</span>(<span class="params">lst</span>):</span><br><span class="line">    <span class="keyword">return</span> lst</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    lst = <span class="built_in">list</span>(<span class="built_in">range</span>(<span class="number">10</span>))</span><br><span class="line">    <span class="built_in">print</span>(yield_func(lst))</span><br><span class="line">    <span class="built_in">print</span>(for_return_func(lst))</span><br><span class="line">    <span class="built_in">print</span>(return_func(lst))</span><br><span class="line">    <span class="keyword">for</span> x <span class="keyword">in</span> yield_func(lst):</span><br><span class="line">        <span class="built_in">print</span>(x, end=<span class="string">&quot; &quot;</span>)</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 输出如下</span></span><br><span class="line">&lt;generator object yield_func at 0x000001644AF624F8&gt;</span><br><span class="line">0</span><br><span class="line">[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]</span><br><span class="line">0 1 2 3 4 5 6 7 8 9</span><br></pre></td></tr></table></figure>
+<p>所以肯定是要利用这个方法来帮助优化程序。在Python操作文件的时候
+<code>readline()</code>函数就是一个生成器，一行一行的返回，查找速度比直接
+<code>read()</code>或者 <code>readlines()</code>更快。</p>
+<p>但这里的问题是，输出的Pi文本是一行。。。所以没法直接使用。当然也可以手动把这个文本按行切割后，再用
+<code>readline()</code>函数。</p>
+<p>这里我找到的解决方案[1]可以更方便的解决这个问题，详见博客or项目源码。</p>
+<h4 id="bug">bug</h4>
+<p>把程序写好之后，可以发现一个很明显的bug，就是查询的字符串如果在两个
+<code>chunk</code>之间，是会直接跳过。比如
+<code>12345678</code>，<code>block_size=4</code>，那么就是
+<code>1234</code>和 <code>5678</code>。如果查找
+<code>45</code>是找不到的。</p>
+<p>所以这里需要做两个额外的处理</p>
+<ol type="1">
+<li>加上上一个 <code>block_size</code>的字符串
+<code>old_chunk = copy.deepcopy(chunk[-len(target_str) :])</code>。这里为了省空间和时间，只加上了需要查询字符串的长度，就可以满足需求</li>
+<li>返回查找位数时需要额外
+<code>-len(old_chunk)</code>。（注：这里一开始忘了操作，故前几位查询的既有会发现结果与北大数院查询的结果差10位，在这里致歉）</li>
+</ol>
+<p>最后经过测试后，性能基本达到预期（见项目README，有说明具体结果）。</p>
+<h2 id="部署">部署</h2>
+<p>至此，核心程序已经完成，接下来需要考虑一种方式来给其他人方便使用了。最直接的其实就是写一个Web，前端去调用后端返回结果。但同样的，这里还是考虑到网站流量等原因，跟北大数院一样放到公众号上也很方便，开发上也只需要完成后端即可，无需考虑前端UI啥的。</p>
+<p>查询公众号文档后，一通操作，就把功能完成了（文案直接"抄"了北大数院的文案）。代码见<a
+href="https://github.com/wnma3mz/birthday_in_pi/blob/master/flask_server.py">flask_server.py</a></p>
+<p>这里需要在微信公众号后台配置开发者模式，然后把token复制进代码。</p>
+<h3 id="tip">Tip</h3>
+<p>公众号开发要求是80端口，但这里通过子域名的方式+nginx反代就可以直接用域名访问，而无需80端口。nginx配置见下</p>
+<figure class="highlight nginx"><table><tr><td class="code"><pre><span class="line"><span class="section">server</span> &#123;</span><br><span class="line">    <span class="attribute">listen</span>       <span class="number">80</span>;</span><br><span class="line">    <span class="attribute">server_name</span>  xxx.yyy.zzz;</span><br><span class="line"></span><br><span class="line">    <span class="section">location</span> / &#123;</span><br><span class="line">        <span class="attribute">uwsgi_pass</span> <span class="number">0.0.0.0</span>:port;</span><br><span class="line">        <span class="attribute">proxy_pass</span> http://0.0.0.0:port/wx_api;</span><br><span class="line">    &#125;</span><br><span class="line">  </span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
+<p>这里的 <code>xxx.yyy.zzz</code>就是域名，port就是
+<code>flask</code>开启的端口</p>
+<h2 id="其他">其他</h2>
+<h3 id="查表">查表</h3>
+<p>当然，查表的方式还是最快的，所以还是建了一个表，并且做了一个简单的分析。分析内容如下</p>
+<p>最近只需246位能够找到,
+最远需要近9亿位。列出Top10（左侧为年月日，右侧为第x位）</p>
+<p>日期范围为19700101-20091231</p>
+<table>
+<thead>
+<tr>
+<th>日期</th>
+<th>位数</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>20190914</td>
+<td>246</td>
+</tr>
+<tr>
+<td>20190516</td>
+<td>5364</td>
+</tr>
+<tr>
+<td>19910403</td>
+<td>8365</td>
+</tr>
+<tr>
+<td>20141020</td>
+<td>9813</td>
+</tr>
+<tr>
+<td>19950728</td>
+<td>1,3874</td>
+</tr>
+<tr>
+<td>20121031</td>
+<td>1,4538</td>
+</tr>
+<tr>
+<td>19990814</td>
+<td>2,5593</td>
+</tr>
+<tr>
+<td>19810705</td>
+<td>2,7202</td>
+</tr>
+<tr>
+<td>19921122</td>
+<td>3,0884</td>
+</tr>
+<tr>
+<td>19930509</td>
+<td>3,6615</td>
+</tr>
+<tr>
+<td>..........</td>
+<td>..........</td>
+</tr>
+<tr>
+<td>19941227</td>
+<td>7,1993,2960</td>
+</tr>
+<tr>
+<td>20080923</td>
+<td>7,2930,2729</td>
+</tr>
+<tr>
+<td>19911201</td>
+<td>7,2985,0775</td>
+</tr>
+<tr>
+<td>20071206</td>
+<td>7,3782,1997</td>
+</tr>
+<tr>
+<td>20090220</td>
+<td>7,5402,4215</td>
+</tr>
+<tr>
+<td>19730829</td>
+<td>7,7357,0338</td>
+</tr>
+<tr>
+<td>20070526</td>
+<td>7,8655,8038</td>
+</tr>
+<tr>
+<td>19951122</td>
+<td>8,2582,0139</td>
+</tr>
+<tr>
+<td>20140407</td>
+<td>8,7320,8091</td>
+</tr>
+<tr>
+<td>19730521</td>
+<td>8,8950,7619</td>
+</tr>
+</tbody>
+</table>
+<p>为什么4位分隔，而不是3位？</p>
+<p>因为我觉得这样看的更舒服，读万、亿更方便。。。</p>
+<h3 id="程序打包">程序打包</h3>
+<p>为方便没有Python程序的用户，同时也不想使用公众号。在Windows上打包了一份exe程序。命令也很简单
+<code>pyinstaller -F .\read_large_pi.py</code>。</p>
+<p>最后的最后，该程序尚存在优化空间，当查询数字较长（超过8位）时，时间花费也更久（超过1s）。</p>
+<h2 id="参考文章">参考文章</h2>
+<p>[1] <a
+href="https://lovemma.github.io/posts/python-%E5%A6%82%E4%BD%95%E6%B5%81%E5%BC%8F%E8%AF%BB%E5%8F%96%E5%A4%A7%E6%96%87%E4%BB%B6/">python-如何流式读取大文件</a></p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>微信开发</tag>
+        <tag>Steaming</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Python3的Tools-scripts整理</title>
+    <url>/hexo_blog/2020/06/14/Python/Python3%E7%9A%84Tools-scripts%E6%95%B4%E7%90%86/</url>
+    <content><![CDATA[<p>Python3的Tools-scripts整理</p>
+<span id="more"></span>
+<p>文件路径为<code>Anaconda3\Tools\scripts</code></p>
+<h3 id="byext.py">byext.py</h3>
+<p>分析文件/文件夹</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># py文件</span></span><br><span class="line">ext  <span class="built_in">bytes</span>  files  lines  words</span><br><span class="line">.py   <span class="number">2655</span>      <span class="number">1</span>     <span class="number">91</span>    <span class="number">265</span></span><br><span class="line">TOTAL   <span class="number">2655</span>      <span class="number">1</span>     <span class="number">91</span>    <span class="number">265</span></span><br><span class="line">ext  <span class="built_in">bytes</span>  files  lines  words</span><br><span class="line"></span><br><span class="line"><span class="comment"># 二进制文件</span></span><br><span class="line">   ext binary  <span class="built_in">bytes</span>  files</span><br><span class="line">&lt;none&gt;      <span class="number">1</span>  <span class="number">10382</span>      <span class="number">1</span></span><br><span class="line"> TOTAL      <span class="number">1</span>  <span class="number">10382</span>      <span class="number">1</span></span><br><span class="line">   ext binary  <span class="built_in">bytes</span>  files</span><br><span class="line"></span><br><span class="line"><span class="comment"># 文件夹</span></span><br><span class="line">  ext binary  <span class="built_in">bytes</span>   dirs  files  lines  words</span><br><span class="line"> .png      <span class="number">2</span>   <span class="number">2823</span>             <span class="number">2</span></span><br><span class="line">  .py         <span class="number">10471</span>             <span class="number">5</span>    <span class="number">224</span>    <span class="number">647</span></span><br><span class="line"> .txt         <span class="number">17404</span>             <span class="number">3</span>    <span class="number">554</span>    <span class="number">960</span></span><br><span class="line">&lt;<span class="built_in">dir</span>&gt;                    <span class="number">1</span></span><br><span class="line">TOTAL      <span class="number">2</span>  <span class="number">30698</span>      <span class="number">1</span>     <span class="number">10</span>    <span class="number">778</span>   <span class="number">1607</span></span><br><span class="line">  ext binary  <span class="built_in">bytes</span>   dirs  files  lines  words</span><br></pre></td></tr></table></figure>
+<table>
+<thead>
+<tr>
+<th>输出</th>
+<th>主要语法</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>ext</td>
+<td><code>os.path.normcase(os.path.splitext[1])</code></td>
+</tr>
+<tr>
+<td>files</td>
+<td><code>os.path.isfile</code></td>
+</tr>
+<tr>
+<td>bytes</td>
+<td>以二进制(<code>rb</code>)读文件，<code>len</code>的输出</td>
+</tr>
+<tr>
+<td>lines</td>
+<td><code>str(data, 'latin-1').splitlines()</code></td>
+</tr>
+<tr>
+<td>words</td>
+<td><code>len(data.split())</code>的输出</td>
+</tr>
+<tr>
+<td>binary</td>
+<td><code>if *b*'\0' in data</code></td>
+</tr>
+<tr>
+<td>dirs</td>
+<td>os.path.isdir</td>
+</tr>
+<tr>
+<td>lnk</td>
+<td>os.path.islink</td>
+</tr>
+</tbody>
+</table>
+<h3 id="checkpip.py">checkpip.py</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">The latest version of setuptools on PyPI <span class="keyword">is</span> <span class="number">46.2</span><span class="number">.0</span>, but ensurepip has <span class="number">40.8</span><span class="number">.0</span></span><br><span class="line">The latest version of pip on PyPI <span class="keyword">is</span> <span class="number">20.1</span>, but ensurepip has <span class="number">19.0</span><span class="number">.3</span></span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 主要是检查版本</span></span><br><span class="line"><span class="keyword">import</span> ensurepip</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(ensurepip._PROJECTS)</span><br><span class="line"><span class="comment"># [(&#x27;setuptools&#x27;, &#x27;40.8.0&#x27;), (&#x27;pip&#x27;, &#x27;19.0.3&#x27;)]</span></span><br></pre></td></tr></table></figure>
+<h3 id="checkpyc.py">checkpyc.py</h3>
+<p>检查pyc文件是否最新</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> importlib.util</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(importlib.util.MAGIC_NUMBER)</span><br><span class="line"><span class="comment"># b&#x27;B\r\r\n&#x27;</span></span><br></pre></td></tr></table></figure>
+<h3 id="cleanfuture.py">cleanfuture.py</h3>
+<p>修改py文件的future语法</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">errprint</span>(<span class="params">*args</span>):</span><br><span class="line">    strings = <span class="built_in">map</span>(<span class="built_in">str</span>, args)</span><br><span class="line">    msg = <span class="string">&#x27; &#x27;</span>.join(strings)</span><br><span class="line">    <span class="keyword">if</span> msg[-<span class="number">1</span>:] != <span class="string">&#x27;\n&#x27;</span>:</span><br><span class="line">        msg += <span class="string">&#x27;\n&#x27;</span></span><br><span class="line">    sys.stderr.write(msg)</span><br><span class="line">    </span><br><span class="line">errprint(<span class="string">&quot;Usage:&quot;</span>, __doc__)</span><br></pre></td></tr></table></figure>
+<h3 id="crlf.py">crlf.py</h3>
+<p>文件中CRLF用LF替换</p>
+<h3 id="lfcr.py">lfcr.py</h3>
+<p>文件中LF用CRLF替换</p>
+<p>迭代：<code>itertools.count()</code></p>
+<h3 id="reindent.py">reindent.py</h3>
+<p>py文件改为4空格缩进</p>
+<h3 id="untabify.py">untabify.py</h3>
+<p>同上</p>
+<h3 id="texi2html.py">texi2html.py</h3>
+<p>texi to html</p>
+<p><code>import keyword</code></p>
+<p><code>import webbrowser</code></p>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>Scripts</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Python杂谈</title>
+    <url>/hexo_blog/2018/03/14/Python/Python%E6%9D%82%E8%B0%88/</url>
+    <content><![CDATA[<p>理解Python中的GIL(Global Interpreter
+Lock)，以及如何使用多线程和多进程，以及弱引用。</p>
+<span id="more"></span>
+<h2 id="gil">GIL</h2>
+<p>建议阅读文章</p>
+<p><a
+href="http://zhuoqiang.me/python-thread-gil-and-ctypes.html">python
+线程，GIL 和 ctypes</a></p>
+<p><a
+href="https://zhuanlan.zhihu.com/p/20953544">谈谈python的GIL、多线程、多进程</a></p>
+<p>GIL, <code>Global Interpreter Lock</code>，全局解释器锁</p>
+<p>产生原因: 防止多线程并发执行机器码的Mutex。</p>
+<p>好处:
+不需要考虑线程安全问题，在单线程的情况下更快(相对而言)，和C库结合时更加方便</p>
+<p>形象的比方:</p>
+<blockquote>
+<p>我们把整个进程空间看做一个车间，把线程看成是多条不相交的流水线，把线程控制流中的字节码看作是流水线上待处理的物品。Python
+解释器是工人，整个车间仅此一名。操作系统是一只上帝之手，会随时把工人从一条流水线调到另一条——这种“随时”是不由分说的，即不管处理完当前物品与否。</p>
+</blockquote>
+<blockquote>
+<p>若没有 GIL。假设工人正在流水线 A 处理 A1 物品，根据 A1
+的需要将房间温度（一个全局对象）调到了 20
+度。这时上帝之手发动了，工人被调到流水线 B 处理 B1 物品，根据 B1
+的需要又将房间温度调到了 50 度。这时上帝之手又发动了，工人又调回 A
+继续处理 A1。但此时 A1 暴露在了 50 度的环境中，安全问题就此产生了。</p>
+</blockquote>
+<blockquote>
+<p>而 GIL 相当于一条锁链，一旦工人开始处理某条流水线上的物品，GIL
+便会将工人和该流水线锁在一起。而被锁住的工人只会处理该流水线上的物品。就算突然被调到另一条流水线，他也不会干活，而是干等至重新调回原来的流水线。这样每个物品在被处理的过程中便总是能保证全局环境不会突变。</p>
+</blockquote>
+<p>在解释器解释python代码时，首先会获得这把锁，需要I/O操作就是释放这把锁。如果是纯计算程序，没有I/O操作，解释器会每隔100次操作释放一次锁（Python3.2之后使用固定时间(5ms)来释放）。</p>
+<p>GIL只会影响严重依赖CPU的程序（计算型）,
+所以运行Python的时候，同一时间只能执行一个线程(单核CPU下的多线程都只是并发)。运行Python程序就只能占用一个核的CPU资源，哪怕使用了
+<code>threading</code>多线程也是一样。</p>
+<p>CPU密集性代码(循环)：不适用多线程。不断触发GIL，线程来回切换，消耗资源</p>
+<p>IO密集型代码(文件处理，网路爬虫)：多线程能够提高效率。在一个线程等待的同时，切换到另一个线程，不浪费CPU资源从而提升效率。</p>
+<p>多核多线程比单核多线程更差，单核下多线程每次释放GIL，唤醒的线程都能获取GIL锁，可以无缝执行。但是多核下，一个CPU释放完GIL之后，所有CPU的线程又开始竞争，可能会被原来的CPU获取，导致其他CPU线程唤醒之后必须等待到下一次释放。这样造成的结果称为线程颠簸(thrashing),
+效率会更低下。</p>
+<p>以上，如果Python需要利用多核CPU，就使用多进程(<code>multiprocessing</code>)，各自拥有独立的GIL，互不干扰，从而达到并行执行。还有一种方案就是使用C进行拓展，手动释放GIL，这里不做介绍。</p>
+<p>参考代码</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-   </span></span><br><span class="line"><span class="keyword">from</span> time <span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">from</span> threading <span class="keyword">import</span> Thread</span><br><span class="line"><span class="keyword">from</span> multiprocessing <span class="keyword">import</span> Process</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">spawn_n_threads</span>(<span class="params">n, target</span>):</span><br><span class="line">    threads = []</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        thread = Thread(target=target)</span><br><span class="line">        thread.start()</span><br><span class="line">        threads.append(thread)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> thread <span class="keyword">in</span> threads:</span><br><span class="line">        thread.join()</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">spawn_n_processes</span>(<span class="params">n, target</span>):</span><br><span class="line">    threads = []</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">        thread = Process(target=target)</span><br><span class="line">        thread.start()</span><br><span class="line">        threads.append(thread)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> thread <span class="keyword">in</span> threads:</span><br><span class="line">        thread.join()</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">test</span>(<span class="params">target, number=<span class="number">10</span>, spawner=spawn_n_threads</span>):</span><br><span class="line">    <span class="keyword">for</span> n <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">5</span>):</span><br><span class="line">        start_time = time()</span><br><span class="line">        <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(number):</span><br><span class="line">            spawner(n, target)</span><br><span class="line">        end_time = time()</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Time elapsed with &#123;&#125; branch(es): &#123;:.6f&#125; sec(s)&quot;</span>.<span class="built_in">format</span>(n, end_time - start_time))</span><br><span class="line"> </span><br><span class="line"><span class="keyword">def</span> <span class="title function_">fib</span>():</span><br><span class="line">    a = b = <span class="number">1</span></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">100000</span>):</span><br><span class="line">        a, b = b, a + b</span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;threads:&quot;</span>)</span><br><span class="line">    test(fib)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;processes:&quot;</span>)</span><br><span class="line">    test(fib, spawner=spawn_n_processes)</span><br></pre></td></tr></table></figure>
+<p>P.S.</p>
+<ul>
+<li>并行：同时处理两个及以上的事件在同一时刻发生</li>
+<li>并发：同时处理两个及以上的事件在同一事件间隔内发生</li>
+</ul>
+<h2 id="弱引用">弱引用</h2>
+<p>推荐阅读</p>
+<p><a
+href="http://blog.soliloquize.org/2016/01/21/Python%E5%BC%B1%E5%BC%95%E7%94%A8%E7%9A%84%E4%BD%BF%E7%94%A8%E4%B8%8E%E6%B3%A8%E6%84%8F%E4%BA%8B%E9%A1%B9/">Python弱引用的使用与注意事项</a></p>
+<p>弱引用与强引用相对，是指不能确保其引用的对象不会被垃圾回收器回收的引用。若一个对象若只被弱引用所引用，则被认为是不可访问（或弱可访问）的，病因此可能在任何时刻被回收。主要作用就是减少循环引用，减少内存中不必要的对象存在的数量。</p>
+<p>Python用
+<code>weakref</code>模块创建对象的弱引用，当引用计数为0或只存在对象的弱引用时将回收这个对象。</p>
+<p>内存管理时使用，cache编程，垃圾回收(GC)</p>
+<h2 id="while-1与while-true">while 1与while True</h2>
+<p><a
+href="http://www.pythoner.com/356.html">Python天坑系列（一）：while
+1比while True更快？</a></p>
+<p>在python2中while 1会比while True更快</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">while_one</span>():</span><br><span class="line">    i = <span class="number">0</span></span><br><span class="line">    <span class="keyword">while</span> <span class="number">1</span>:</span><br><span class="line">        i += <span class="number">1</span></span><br><span class="line">        <span class="keyword">if</span> i == <span class="number">10000000</span>:</span><br><span class="line">            <span class="keyword">break</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">while_true</span>():</span><br><span class="line">    i = <span class="number">0</span></span><br><span class="line">    <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">        i += <span class="number">1</span></span><br><span class="line">        <span class="keyword">if</span> i == <span class="number">10000000</span>:</span><br><span class="line">            <span class="keyword">break</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    w1_time, wt_time = <span class="number">0</span>, <span class="number">0</span></span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">        w1 = timeit.timeit(while_one, <span class="string">&quot;from __main__ import while_one&quot;</span>, number=<span class="number">3</span>)</span><br><span class="line">        w1_time += w1</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10</span>):</span><br><span class="line">        wt = timeit.timeit(while_true, <span class="string">&quot;from __main__ import while_true&quot;</span>, number=<span class="number">3</span>)</span><br><span class="line">        wt_time += wt</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;while one: %s\nwhile_true: %s&quot;</span> % (w1_time, wt_time))</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2的输出</span></span><br><span class="line"><span class="keyword">while</span> one: 9.27971386909</span><br><span class="line">while_true: 13.5762348175</span><br><span class="line"><span class="comment"># python3的输出</span></span><br><span class="line"><span class="keyword">while</span> one: 14.19957690299998</span><br><span class="line">while_true: 13.785004297995329</span><br></pre></td></tr></table></figure>
+<p>原因：python2中True/False不是关键字，python3添加了True/False做为关键字。每次在循环的时候，True/False需要被检查，所以while
+1会更快</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">Python <span class="number">2.7</span><span class="number">.14</span> (default, Sep <span class="number">23</span> <span class="number">2017</span>, <span class="number">22</span>:06:<span class="number">14</span>)</span><br><span class="line">[GCC <span class="number">7.2</span><span class="number">.0</span>] on linux2</span><br><span class="line"><span class="type">Type</span> <span class="string">&quot;help&quot;</span>, <span class="string">&quot;copyright&quot;</span>, <span class="string">&quot;credits&quot;</span> <span class="keyword">or</span> <span class="string">&quot;license&quot;</span> <span class="keyword">for</span> more information.</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span><span class="keyword">import</span> keyword</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>keyword.kwlist</span><br><span class="line">[<span class="string">&#x27;and&#x27;</span>, <span class="string">&#x27;as&#x27;</span>, <span class="string">&#x27;assert&#x27;</span>, <span class="string">&#x27;break&#x27;</span>, <span class="string">&#x27;class&#x27;</span>, <span class="string">&#x27;continue&#x27;</span>, <span class="string">&#x27;def&#x27;</span>, <span class="string">&#x27;del&#x27;</span>, <span class="string">&#x27;elif&#x27;</span>, <span class="string">&#x27;else&#x27;</span>, <span class="string">&#x27;except&#x27;</span>, <span class="string">&#x27;exec&#x27;</span>, <span class="string">&#x27;finally&#x27;</span>, <span class="string">&#x27;for&#x27;</span>, <span class="string">&#x27;from&#x27;</span>, <span class="string">&#x27;global&#x27;</span>, <span class="string">&#x27;if&#x27;</span>, <span class="string">&#x27;import&#x27;</span>, <span class="string">&#x27;in&#x27;</span>, <span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;lambda&#x27;</span>, <span class="string">&#x27;not&#x27;</span>, <span class="string">&#x27;or&#x27;</span>, <span class="string">&#x27;pass&#x27;</span>, <span class="string">&#x27;print&#x27;</span>, <span class="string">&#x27;raise&#x27;</span>, <span class="string">&#x27;return&#x27;</span>, <span class="string">&#x27;try&#x27;</span>, <span class="string">&#x27;while&#x27;</span>, <span class="string">&#x27;with&#x27;</span>, <span class="string">&#x27;yield&#x27;</span>]</span><br><span class="line"></span><br><span class="line">Python <span class="number">3.6</span><span class="number">.3</span> (default, Oct  <span class="number">3</span> <span class="number">2017</span>, <span class="number">21</span>:<span class="number">45</span>:<span class="number">48</span>)</span><br><span class="line">[GCC <span class="number">7.2</span><span class="number">.0</span>] on linux</span><br><span class="line"><span class="type">Type</span> <span class="string">&quot;help&quot;</span>, <span class="string">&quot;copyright&quot;</span>, <span class="string">&quot;credits&quot;</span> <span class="keyword">or</span> <span class="string">&quot;license&quot;</span> <span class="keyword">for</span> more information.</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span><span class="keyword">import</span> keyword</span><br><span class="line"><span class="meta">&gt;&gt;&gt; </span>keyword.kwlist</span><br><span class="line">[<span class="string">&#x27;False&#x27;</span>, <span class="string">&#x27;None&#x27;</span>, <span class="string">&#x27;True&#x27;</span>, <span class="string">&#x27;and&#x27;</span>, <span class="string">&#x27;as&#x27;</span>, <span class="string">&#x27;assert&#x27;</span>, <span class="string">&#x27;break&#x27;</span>, <span class="string">&#x27;class&#x27;</span>, <span class="string">&#x27;continue&#x27;</span>, <span class="string">&#x27;def&#x27;</span>, <span class="string">&#x27;del&#x27;</span>, <span class="string">&#x27;elif&#x27;</span>, <span class="string">&#x27;else&#x27;</span>, <span class="string">&#x27;except&#x27;</span>, <span class="string">&#x27;finally&#x27;</span>, <span class="string">&#x27;for&#x27;</span>, <span class="string">&#x27;from&#x27;</span>, <span class="string">&#x27;global&#x27;</span>, <span class="string">&#x27;if&#x27;</span>, <span class="string">&#x27;import&#x27;</span>, <span class="string">&#x27;in&#x27;</span>, <span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;lambda&#x27;</span>, <span class="string">&#x27;nonlocal&#x27;</span>, <span class="string">&#x27;not&#x27;</span>, <span class="string">&#x27;or&#x27;</span>, <span class="string">&#x27;pass&#x27;</span>, <span class="string">&#x27;raise&#x27;</span>, <span class="string">&#x27;return&#x27;</span>, <span class="string">&#x27;try&#x27;</span>, <span class="string">&#x27;while&#x27;</span>, <span class="string">&#x27;with&#x27;</span>, <span class="string">&#x27;yield&#x27;</span>]</span><br></pre></td></tr></table></figure>
+<h2 id="if-x-true-与-if-x">if x == True 与 if x</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment">#! /usr/bin/python</span></span><br><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> timeit</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">if_x_eq_true</span>():</span><br><span class="line">    x = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">if</span> x == <span class="literal">True</span>:</span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">if_x</span>():</span><br><span class="line">    x = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">if</span> x:</span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    if1 = timeit.timeit(if_x_eq_true, <span class="string">&quot;from __main__ import if_x_eq_true&quot;</span>, number = <span class="number">1000000</span>)</span><br><span class="line">    if2 = timeit.timeit(if_x, <span class="string">&quot;from __main__ import if_x&quot;</span>, number = <span class="number">1000000</span>)</span><br><span class="line"></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;if_x_eq_true: %s\nif_x: %s&quot;</span> % (if1, if2))</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2</span></span><br><span class="line">if_x_eq_true: 0.118808984756</span><br><span class="line">if_x: 0.0920550823212</span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line">if_x_eq_true: 0.0951261969985353</span><br><span class="line">if_x: 0.07899275699855934</span><br></pre></td></tr></table></figure>
+<p>不管python2还是python3, <code>if x:</code>从各方面来说都会优于
+<code>if x == True</code>。后者会比前者多出一个检查True值和比较的操作。</p>
+<h2 id="type与object">type与object</h2>
+<p><a href="https://www.zhihu.com/question/38791962">Python 的 type 和
+object 之间是怎么一种关系？</a></p>
+<h2
+id="深刻理解python中的元类metaclass">深刻理解Python中的元类(metaclass)</h2>
+<p><a
+href="http://blog.jobbole.com/21351/">深刻理解Python中的元类(metaclass)</a></p>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>GIL</tag>
+        <tag>元类</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Python 自动导入包</title>
+    <url>/hexo_blog/2023/12/03/Python/Python%E8%87%AA%E5%8A%A8%E5%AF%BC%E5%85%A5%E5%8C%85/</url>
+    <content><![CDATA[<p>在一个文件夹中，有很多个 Python
+文件，每个文件都有若干重名函数，需要一一导入。这个时候，可以用 Python
+的自动导入包来实现。</p>
+<span id="more"></span>
+<h2 id="需求描述">需求描述</h2>
+<p>假设有一个文件夹</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">func/</span><br><span class="line">├── __init__.py</span><br><span class="line">├── func1.py</span><br><span class="line">├── func2.py</span><br><span class="line">├── func3.py</span><br></pre></td></tr></table></figure>
+<p>每个文件都有一个重名函数，比如</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># func1.py</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">foo</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;foo&quot;</span>)</span><br></pre></td></tr></table></figure>
+<p>这不仅导致 <code>import</code> 代码不够简洁，且在新增文件时，还得在
+<code>__init__.py</code> 中添加 <code>import</code>
+语句。甚至在调用处也得增加代码来处理。</p>
+<h2 id="解决方案">解决方案</h2>
+<p>Python 的自动导入包可以解决这个问题。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> glob</span><br><span class="line"><span class="keyword">import</span> importlib</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取当前文件夹下所有的 .py 文件</span></span><br><span class="line">files = glob.glob(os.path.join(os.path.dirname(__file__), <span class="string">&quot;*.py&quot;</span>))</span><br><span class="line">func_dict = &#123;&#125;</span><br><span class="line"><span class="keyword">for</span> file_ <span class="keyword">in</span> files:</span><br><span class="line">    func_name = os.path.basename(file_)[:-<span class="number">3</span>]</span><br><span class="line">    load_module = importlib.import_module(<span class="string">f&quot;func.<span class="subst">&#123;func_name&#125;</span>&quot;</span>)</span><br><span class="line">    <span class="built_in">globals</span>()[load_module.__name__] = <span class="built_in">getattr</span>(load_module, <span class="string">&quot;foo&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 将其放到一个字典中，以按需处理，</span></span><br><span class="line">    func_dict[func_name] = <span class="built_in">globals</span>()[load_module.__name__]</span><br></pre></td></tr></table></figure>
+<p>这样，不仅不需要在 <code>func/__init__.py</code> 中添加
+<code>import</code>
+语句，也不需要在调用处添加额外的代码。甚至，还要根据需求，从字典中调用对应函数。当然，这需要提前进行约定，比如全部放到<code>func</code>文件夹下，且都用<code>foo</code>这个名字作为函数名。</p>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>importlib</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Python装饰器</title>
+    <url>/hexo_blog/2018/01/30/Python/Python%E8%A3%85%E9%A5%B0%E5%99%A8/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Python的装饰器是什么和一些Python内置装饰器的使用</p>
+<span id="more"></span>
+<p>参考文章:</p>
+<p><a
+href="https://www.zhihu.com/question/26930016">如何理解Python装饰器？</a></p>
+<p><a
+href="https://www.cnblogs.com/cicaday/p/python-decorator.html">详解Python的装饰器</a></p>
+<h2 id="装饰器是什么">装饰器是什么</h2>
+<p>python的装饰器本身就是一个函数，使用它可以在某些场景下简化python代码，增加一些额外功能。我的理解是，python装饰器可以帮助同时多个函数做一样的事情（比如打印日志），而避免了每个函数都需要额外加上一样的代码。</p>
+<p>举个例子，假设我们有两个函数</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
+<p>以上一段代码片段很容易理解，运行结果就是两行输出。那么问题来了，如果这个时候我需要在每个函数里面加上一个输出，比如<code>print("this is func")</code>。很简单，我们想到了第一种解决方案就是同时在两个函数里面加上这么一行。</p>
+<p>但是如果我们需要在很多个函数在上很多个这种重复代码，我们这么做就很浪费时间了，而且容易出错。这个时候我们想到了第二种解决方案。定义一个新的函数，在每次使用函数之前，先调用新函数。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">    func()</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_print(func_one)</span><br><span class="line">    func_print(func_two)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<p>感觉不错，但是如果这样无疑改变了原来的逻辑结构，且每次需要传递函数给<code>func_print</code>。下面就轮到我们的主角登场了——装饰器。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">        <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">    <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one = func_print(func_one)</span><br><span class="line">    func_two = func_print(func_two)</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
+<p>这样也能达到我们的想要的效果，但是比之前的实现方式更难看懂。而且调用起来似乎更复杂了。</p>
+<p>第一个问题：更难看懂</p>
+<p>可能是对<code>*args</code>、<code>**kwargs</code>的参数理解不是很好。简单解释一下就是说,<code>*args</code>可以理解为一个<code>list</code>，接收多个参数。<code>**kwargs</code>可以理解为一个<code>dict</code>，接收多个<code>key</code>和<code>value</code>。接收的参数数量不定，可以没有，也可以很多个。再具体一点可以百度看看。</p>
+<p>第二个问题：调用更复杂</p>
+<p>其实上面的例子其实调用装饰器的一种方式，<code>python</code>有<code>@</code>的这个语法糖，所以一般情况下，我们是像下面一样调用的。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;this is func&quot;</span>)</span><br><span class="line">        <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">    <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br></pre></td></tr></table></figure>
+<p>好的，简单的装饰器调用基本就是这样，难度进阶，如何使用带参数的装饰器。换句话就说，如果我们需要根据每个函数自定义一些东西，如何传递参数给装饰器。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_print</span>(<span class="params">func_name</span>):</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">decorator</span>(<span class="params">func</span>):</span><br><span class="line"></span><br><span class="line">        <span class="keyword">def</span> <span class="title function_">wrapper</span>(<span class="params">*args, **kwargs</span>):</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;this is %s&quot;</span> % func_name)</span><br><span class="line">            <span class="keyword">return</span> func(*args, **kwargs)</span><br><span class="line">        <span class="keyword">return</span> wrapper</span><br><span class="line"></span><br><span class="line">    <span class="keyword">return</span> decorator</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print(<span class="params"><span class="string">&quot;one&quot;</span></span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_one</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_one&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@func_print(<span class="params">func_name=<span class="string">&quot;two&quot;</span></span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">func_two</span>():</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;this is func_two&quot;</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    func_one()</span><br><span class="line">    func_two()</span><br><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<p>观察<code>func_print</code>部分代码，与之前的不同在于多加了一个<code>decorator</code>函数封装了<code>wrapper</code>函数。我的理解是这样的：<code>func_print</code>作用是装饰器的名字和接收外部的参数，<code>decorator</code>接收外部的函数，<code>wrapper</code>返回结果给函数。<code>decorator</code>翻译为中文是<strong>装饰</strong>。<code>wrapper</code>翻译过来是<strong>包装</strong>。<code>decorator</code>装饰函数，<code>wrapper</code>包装函数。</p>
+<p>当然，<code>decorator</code>和<code>wrapper</code>这两个函数名字可以自定义，在这里为了好理解所以用了这两个名字</p>
+<h2 id="内置装饰器">内置装饰器</h2>
+<h3 id="property"><code>@property</code></h3>
+<p>如果要了解使用这个装饰器，就需要知道在不用装饰器的情况下，写一个属性。这里使用python自带的一个例子。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">C</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">getx</span>(<span class="params">self</span>): <span class="keyword">return</span> <span class="variable language_">self</span>._x</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">setx</span>(<span class="params">self, value</span>): <span class="variable language_">self</span>._x = value</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">delx</span>(<span class="params">self</span>): <span class="keyword">del</span> <span class="variable language_">self</span>._x</span><br><span class="line">    x = <span class="built_in">property</span>(getx, setx, delx, <span class="string">&quot;I&#x27;m the &#x27;x&#x27; property.&quot;</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<p>上面就是定义一个python属性的标准写法，下面使用<code>@property</code>来定义属性</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">C</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line"><span class="meta">    @property</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="string">&quot;I am the &#x27;x&#x27; property.&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>._x</span><br><span class="line"><span class="meta">    @x.setter</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>._x = value</span><br><span class="line"><span class="meta">    @x.deleter</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">x</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>._x</span><br></pre></td></tr></table></figure>
+<p>这样有什么好处呢，原来的定义，属性访问并没有做限制。使用<code>@property</code>可以使<code>x</code>这个属性更加安全，从获取、设置必须与属性名一致。</p>
+<p>Emmmm.....我的理解就是加上<code>@property</code>的话，定义属性会更加安全更加优雅。毕竟是面向对象了，天知道你的用户会对你的属性做一些什么奇奇怪怪的事情。</p>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>Decorators</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>多进程嵌套内存泄漏问题记录</title>
+    <url>/hexo_blog/2023/11/01/Python/%E5%A4%9A%E8%BF%9B%E7%A8%8B%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F%E9%97%AE%E9%A2%98%E8%AE%B0%E5%BD%95/</url>
+    <content><![CDATA[<p>记一次多进程嵌套发现的内存泄漏问题，以及解决方案。</p>
+<span id="more"></span>
+<h3 id="问题代码">问题代码</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> multiprocessing</span><br><span class="line"><span class="keyword">from</span> concurrent.futures <span class="keyword">import</span> ProcessPoolExecutor</span><br><span class="line"><span class="keyword">import</span> concurrent.futures</span><br><span class="line"><span class="keyword">from</span> typing <span class="keyword">import</span> <span class="type">Any</span></span><br><span class="line"><span class="keyword">import</span> gc</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Foo</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="variable language_">self</span>.executor = ProcessPoolExecutor(max_workers=<span class="number">10</span>, mp_context=multiprocessing.get_context(<span class="string">&quot;spawn&quot;</span>))</span><br><span class="line"></span><br><span class="line"><span class="meta">    @staticmethod</span></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">fun</span>(<span class="params">arg</span>):</span><br><span class="line">        <span class="keyword">raise</span> concurrent.futures.process._RemoteTraceback(<span class="string">&quot;error&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">call</span>(<span class="params">self, i</span>):</span><br><span class="line">        x = []</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">            res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line">    <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;-&quot;</span>*<span class="number">20</span>)</span><br><span class="line">        <span class="keyword">return</span> x</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    <span class="keyword">with</span> ProcessPoolExecutor(max_workers=<span class="number">2</span>, mp_context=multiprocessing.get_context(<span class="string">&quot;spawn&quot;</span>)) <span class="keyword">as</span> e:</span><br><span class="line">        futures = e.submit(predict)</span><br><span class="line">        futures.result()</span><br></pre></td></tr></table></figure>
+<p>肉眼观察 乍一看这段代码是没有任何问题的，在 submit 之后，获取 result
+，并且用 try ... except ... 来捕获可能出现的任何情况。尤其是，当不使用
+main 函数中的进程池时，直接普通的使用 predict
+函数，这段代码不会引发任何问题，可以正常的退出。</p>
+<p>但为什么改成多进程时，这段代码就无法正常退出了呢？程序无法正常退出往往是因为程序某些对象没有完全被释放干净，但这里并没有使用特殊的函数却依旧引发了该问题，所以需要借助工具来进行简单的分析。</p>
+<h3 id="分析">分析</h3>
+<p>garbage collector python 的垃圾回收机制主要使用引用计数（reference
+counting）来跟踪和回收垃圾。大部分情况下，Python
+能够在对象结束执行后，引用计数会变为
+0，使得最后能够被回收。这里程序退出不了的原因是某些对象没发自动退出，所以在
+predict 函数中，观察变量 f 和 f.executor 的引用计数</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f.executor))</span><br></pre></td></tr></table></figure>
+<p>程序的输出，对象 f 的计数为 5，而 f.executor 的计数为 2。为什么是
+2，因此在最外层还有一个进程数量为 2 的进程池没有执行完。而 f 等于 4
+而不等于 f.executor 这就不符合预期，比正常数值多 3。这个 3 是因为循环了
+3 次，每次都引发了异常。</p>
+<p>进一步地，观察 f 中什么对象没有被清理。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">predict</span>():</span><br><span class="line">    f = Foo()</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">3</span>):</span><br><span class="line">        f.call(i)</span><br><span class="line">    <span class="built_in">print</span>(gc.get_referrers(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f))</span><br><span class="line">    <span class="built_in">print</span>(sys.getrefcount(f.executor))</span><br></pre></td></tr></table></figure>
+<p>此时输出会多一行，形如</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">[&lt;frame at 0x10c266240, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;, &lt;frame at 0x10c6b0240, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;, &lt;frame at 0x10c6b0440, file <span class="string">&#x27;XXX&#x27;</span>, line 24, code call&gt;]</span><br></pre></td></tr></table></figure>
+<p>这表明程序在何处的对象没有被处理，可以定位到函数 call 的 return
+位置。</p>
+<p>为什么？ 在获取 result 时，如果遇到异常，则中间变量 future
+无法被回收。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">result</span>(<span class="params">self, timeout=<span class="literal">None</span></span>):</span><br><span class="line">    <span class="string">&quot;&quot;&quot;Return the result of the call that the future represents.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Args:</span></span><br><span class="line"><span class="string">        timeout: The number of seconds to wait for the result if the future</span></span><br><span class="line"><span class="string">            isn&#x27;t done. If None, then there is no limit on the wait time.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Returns:</span></span><br><span class="line"><span class="string">        The result of the call that the future represents.</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">    Raises:</span></span><br><span class="line"><span class="string">        CancelledError: If the future was cancelled.</span></span><br><span class="line"><span class="string">        TimeoutError: If the future didn&#x27;t finish executing before the given</span></span><br><span class="line"><span class="string">            timeout.</span></span><br><span class="line"><span class="string">        Exception: If the call raised then that exception will be raised.</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        <span class="keyword">with</span> <span class="variable language_">self</span>._condition:</span><br><span class="line">            <span class="keyword">if</span> <span class="variable language_">self</span>._state <span class="keyword">in</span> [CANCELLED, CANCELLED_AND_NOTIFIED]:</span><br><span class="line">                <span class="keyword">raise</span> CancelledError()</span><br><span class="line">            <span class="keyword">elif</span> <span class="variable language_">self</span>._state == FINISHED:</span><br><span class="line">                <span class="keyword">return</span> <span class="variable language_">self</span>.__get_result()</span><br><span class="line"></span><br><span class="line">    <span class="variable language_">self</span>._condition.wait(timeout)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> <span class="variable language_">self</span>._state <span class="keyword">in</span> [CANCELLED, CANCELLED_AND_NOTIFIED]:</span><br><span class="line">                <span class="keyword">raise</span> CancelledError()</span><br><span class="line">            <span class="keyword">elif</span> <span class="variable language_">self</span>._state == FINISHED:</span><br><span class="line">                <span class="keyword">return</span> <span class="variable language_">self</span>.__get_result()</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                <span class="keyword">raise</span> TimeoutError()</span><br><span class="line">    <span class="keyword">finally</span>:</span><br><span class="line">        <span class="comment"># Break a reference cycle with the exception in self._exception</span></span><br><span class="line">        <span class="variable language_">self</span> = <span class="literal">None</span></span><br></pre></td></tr></table></figure>
+<h3 id="解决方案">解决方案</h3>
+<h4 id="消除中间变量-future">消除中间变量 future</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line"><span class="comment"># 改为</span></span><br><span class="line">res = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i).result(timeout=<span class="number">1</span>)</span><br></pre></td></tr></table></figure>
+<h4 id="手动-gc">手动 GC</h4>
+<p>在调用完 Foo 之后，使用 gc.collect()</p>
+<h4 id="主动关闭进程池">主动关闭进程池</h4>
+<p>在调用完 Foo 之后，使用 f.executor.shutdown()</p>
+<h4 id="在-return-前-主动删除未释放的对象">在 return 前
+主动删除未释放的对象</h4>
+<p>如下所示，使用 future = None 也可以达到相似效果（让 Python
+自动调用垃圾回收机制）</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">call</span>(<span class="params">self, i</span>):</span><br><span class="line">   x = []</span><br><span class="line">   <span class="keyword">try</span>:</span><br><span class="line">       future = <span class="variable language_">self</span>.executor.submit(<span class="variable language_">self</span>.fun, i)</span><br><span class="line">       res = future.result(timeout=<span class="number">1</span>)</span><br><span class="line">   <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">       <span class="built_in">print</span>(<span class="string">&quot;-&quot;</span>*<span class="number">20</span>)</span><br><span class="line">       <span class="keyword">del</span> future</span><br><span class="line">   <span class="keyword">return</span> x</span><br></pre></td></tr></table></figure>
+<h3 id="q-a">Q &amp; A</h3>
+<p>Q: 能不能在 Foo 中添加 <strong>del</strong> 函数解决这个问题 A: 在
+Python 3.8.16 可以，Python 3.9.6 不行。可以手动增加 <strong>del</strong>
+函数添加 print 函数，观察是否有对应的输出。</p>
+<p>Q: gc 显示为什么是在 return 这，而不是其他地方？ A: frame
+对象有一个属性叫作 f_lineno，它表示当前执行的代码行。当函数执行到 return
+语句时，f_lineno 会更新为 return 语句所在的行号。</p>
+<p>这是因为在函数返回值之后，控制权会返回到调用该函数的代码。此时函数的上下文（包括它的局部变量、执行状态等）会被丢弃，栈帧也会被弹出调用栈。这个过程称为函数的退出或返回。所以，return
+语句所在的行是函数执行的最后一行。</p>
 ]]></content>
       <categories>
-        <category>Database</category>
+        <category>Python</category>
       </categories>
       <tags>
-        <tag>数据库</tag>
-        <tag>SQL</tag>
+        <tag>多进程</tag>
+        <tag>内存泄漏</tag>
       </tags>
   </entry>
   <entry>
-    <title>Hadoop伪分布式安装（Centos7）</title>
-    <url>/hexo_blog/2017/04/09/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Hadoop%E4%BC%AA%E5%88%86%E5%B8%83%E5%BC%8F%E5%AE%89%E8%A3%85%EF%BC%88Centos7%EF%BC%89/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Hadoop的伪分布式安装和一些基础操作</p>
+    <title>海量数据处理问题（HashMap+分治法、位图法）</title>
+    <url>/hexo_blog/2018/04/05/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E6%B5%B7%E9%87%8F%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E9%97%AE%E9%A2%98%EF%BC%88HashMap+%E5%88%86%E6%B2%BB%E6%B3%95%E3%80%81%E4%BD%8D%E5%9B%BE%E6%B3%95%EF%BC%89/</url>
+    <content><![CDATA[<p>从海量数据中找出不重复的数和出现次数最多的数的解决方案，分别采用了分治法+HashMap和位图法Bitmap两种方法。其中，位图法在大数据量下处理速度更快，内存占用更小。而对于其他类似问题，可以尝试使用分治法加最小堆/最大堆的方式处理。</p>
 <span id="more"></span>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/hadoop/wKiom1SFSNnAeaEUAADD3ZcjTjw828.jpg" /></p>
-<h2 id="文章开头">文章开头</h2>
-<p>环境配置：Centos7</p>
-<p>Hadoop版本：hadoop：2.7.3</p>
-<p>JDK版本：jdk-8u111-linux-x64.tar.gz</p>
-<h2 id="安装之前">安装之前</h2>
+<p>原文链接：<a
+href="https://juejin.im/post/5aa0ee9f518825557c010bc0">经典面试问题: Top
+K 之 ---- 海量数据找出现次数最多或，不重复的。</a></p>
+<p>在数据处理中，较为棘手的有这么一种问题，海量数据如何进行处理。一旦数据的数量足够大，我们就不能使用常规的思路来解决。典型例子就是排序算法，在数据不大（不超过内存）的情况下，可以直接使用排序算法，但是当数据量很大时，我们就需要使用如多路归并此类的算法。</p>
+<p>总结问题：</p>
 <ol type="1">
-<li><p>下载hadoop安装包, <a
-href="http://hadoop.apache.org/releases.html">下载网址</a>。进入官网之后，点击对应版本的“binary”，之后点击链接下载即可。</p></li>
-<li><p>jdk。通过官网下载jdk</p></li>
-<li><p>通过Xshell，上传两个安装包至/usr/local/。服务器与本地下载和上传需要lrzsz这个软件</p></li>
+<li>内存一次性不能放下所有的数</li>
+<li>机器CPU核数不够</li>
+<li>……</li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">yum install lrzsz</span><br></pre></td></tr></table></figure>
-<p>之后可手动拖动文件至服务器当前目录下，也可通过命令
-“rz”来打开窗口，从而进行上传。 （下载命令“sz”）</p>
-<h2 id="解压缩及配置">解压缩及配置</h2>
-<p>在<code>/usr/local/</code>目录下进行解压 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#请根据下载版本，进行解压缩</span></span><br><span class="line">tar -zxvf jdk-8u111-linux-x64.tar.gz</span><br><span class="line">tar -zxvf hadoop-2.7.3.tar.gz</span><br></pre></td></tr></table></figure></p>
-<h3
-id="配置java环境变量可与配置hadoop环境变量配置一起进行">配置Java环境变量（可与配置Hadoop环境变量配置一起进行）</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#第十行左右输入命令，注意版本</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/local/jdk1.8.0_111</span><br><span class="line"><span class="built_in">export</span> PATH=.:<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>
-<h3 id="配置hadoop环境变量">配置Hadoop环境变量</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#在刚刚JAVA_HOME之下一行，注意版本</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#此处的PATH承接上面Java环境变量的配置，即是在原来基础上进行增加而不是另起一行</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_HOME=/usr/local/hadoop-2.7.3</span><br><span class="line"><span class="built_in">export</span> PATH=.:<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$HADOOP_HOME</span>/bin:<span class="variable">$HADOOP_HOME</span>/sbin:<span class="variable">$PATH</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#更新设置</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>
-<h3 id="配置启动hadoop">配置启动Hadoop</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入配置文件目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop-2.7.3/etc/hadoop/</span><br><span class="line"><span class="comment">#修改配置文件</span></span><br><span class="line">vim hadoop-env.sh</span><br><span class="line"><span class="comment">#第二十五行</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/local/jdk1.8.0_111</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim core-site.xml</span><br><span class="line"><span class="comment">#添加&lt;configuration&gt;&lt;/configuration&gt;中间的配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 指定HDFS老大（namenode）的通信地址 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">	    &lt;name&gt;fs.defaultFS&lt;/name&gt;</span><br><span class="line">	    &lt;value&gt;hdfs://hadoopbonc1:9000&lt;/value&gt;</span><br><span class="line">    &lt;!-- 指定hadoop运行时产生文件的存储路径 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hadoop.tmp.dir&lt;/name&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#此处fs.defaultFS的vaule写本机主机名，即第一个value的hadoopbonc1换成主机名</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim hdfs-site.xml</span><br><span class="line"><span class="comment">#添加如上配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 设置hdfs副本数量 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">         &lt;name&gt;dfs.replication&lt;/name&gt;</span><br><span class="line">         &lt;value&gt;1&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br></pre></td></tr></table></figure>
-<h3 id="配置ssh免密码登陆密码互通">配置SSH免密码登陆（密码互通）</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#配置前</span></span><br><span class="line">ssh localhost</span><br><span class="line"><span class="comment">#回车后，要求输入本机登陆密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#配置主机名</span></span><br><span class="line">vim /etc/sysconfig/network</span><br><span class="line"><span class="built_in">source</span> /etc/sysconfig/network</span><br><span class="line"></span><br><span class="line"><span class="comment">#配置hosts</span></span><br><span class="line">vim /etc/hosts</span><br><span class="line">ssh-keygen</span><br><span class="line"></span><br><span class="line"><span class="comment">#把公钥和私钥复制到相应的节点（因为本机是单节点，所以复制到本机）</span></span><br><span class="line"><span class="comment">#此处的hadoopbonc1换成主机名</span></span><br><span class="line">ssh-copy-id hadoopbonc1</span><br></pre></td></tr></table></figure>
-<h3 id="hdfs启动与停止">hdfs启动与停止</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#第一次启动需要格式化，理由跟磁盘第一次使用需要格式化一样</span></span><br><span class="line"><span class="comment">#以后启动就不需要格式化</span></span><br><span class="line">hdfs namenode -format</span><br><span class="line"><span class="comment">#启动hdfs</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#在hadoop目录下</span></span><br><span class="line"><span class="built_in">cd</span> /usr/locate/hadoop-2.7.3/</span><br><span class="line"><span class="comment">#启动</span></span><br><span class="line">./sbin/start-dfs.sh</span><br><span class="line"><span class="comment">#输入jps检查是否成功</span></span><br><span class="line">jps</span><br><span class="line"><span class="comment">#出现四行顺序不定，分别为SecondaryNameNode、DataNode、NameNode、Jps，即表示成功</span></span><br><span class="line"><span class="comment">#打开浏览器输入：服务器ip:50070</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#若无响应则需要开放50070端口，则需开放50070端口</span></span><br><span class="line">firewall-cmd --zone=public --add-port=50070/tcp --permanent</span><br><span class="line">firewall-cmd --reload</span><br><span class="line"></span><br><span class="line"><span class="comment">#hdfs的停止</span></span><br><span class="line"><span class="comment">#同样在hadoop目录下</span></span><br><span class="line">./sbin/stop-dfs.sh</span><br><span class="line"></span><br></pre></td></tr></table></figure>
-<h3 id="配置和启动yarn">配置和启动YARN</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#切换配置文件目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop-2.7.3/etc/hadoop/</span><br><span class="line"><span class="built_in">mv</span> mapred-site.xml.template mapred-site.xml</span><br><span class="line">vim mapred-site.xml</span><br><span class="line"><span class="comment">#添加&lt;configuration&gt;&lt;/configuration&gt;中间的配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 通知框架MR使用YARN --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;mapreduce.framework.name&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;yarn&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim yarn-site.xml</span><br><span class="line"><span class="comment">#添加如上配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">	&lt;!-- reducer取数据的方式是mapreduce_shuffle --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">         &lt;name&gt;yarn.nodemanager.aux-services&lt;/name&gt;</span><br><span class="line">         &lt;value&gt;mapreduce_shuffle&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#启动YARN</span></span><br><span class="line">start-yarn.sh</span><br><span class="line"><span class="comment">#输入jps检查是否成功</span></span><br><span class="line">jps</span><br><span class="line"><span class="comment">#出现六行顺序不定，分别为SecondaryNameNode、DataNode、NameNode、Jps、ResourceManager、NodeManager，即表示成功</span></span><br><span class="line"><span class="comment">#打开浏览器输入：服务器ip:8088</span></span><br><span class="line"><span class="comment">#若无响应，可参照上文中的开放50070端口，开放8088端口，只需将50070换为8088即可</span></span><br></pre></td></tr></table></figure>
-<h2 id="测试">测试</h2>
-<p>在本地新建一个文件，如在<code>/home/user/</code>下新建<code>words.txt</code>，内容如下
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">hello world</span><br><span class="line">hello hadoop</span><br><span class="line">hello csdn</span><br><span class="line">hello</span><br></pre></td></tr></table></figure></p>
-<p>正式进行测试。命令如下： <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#在hdfs根目录下新建test目录</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">mkdir</span> /test</span><br><span class="line"></span><br><span class="line"><span class="comment">#查看hdfs根目录下的目录结构</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">ls</span> /</span><br><span class="line"></span><br><span class="line"><span class="comment">#将本地文件上传至/test/目录下</span></span><br><span class="line">bin/hdfs dfs -put /home/user/words.txt /test/</span><br><span class="line"></span><br><span class="line"><span class="comment">#运行wordcount</span></span><br><span class="line">bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar wordcount /test/words.txt /test/out</span><br><span class="line"></span><br><span class="line"><span class="comment">#在/test/目录下生成了一个名为out的文件目录，查看一下/out/目录下的文件</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">ls</span> /test/out</span><br><span class="line"></span><br><span class="line"><span class="comment">#结果保存在part-r-00000，查看一下运行结果</span></span><br><span class="line">bin/hdfs fs -<span class="built_in">cat</span> /test/out/part-r-00000</span><br></pre></td></tr></table></figure></p>
-<h2 id="hdfs的常用操作命令">HDFS的常用操作命令</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#常用操作：</span></span><br><span class="line"><span class="comment">#HDFS shell</span></span><br><span class="line"><span class="comment">#查看帮助</span></span><br><span class="line">hadoop fs -<span class="built_in">help</span> &lt;cmd&gt;</span><br><span class="line"><span class="comment">#上传</span></span><br><span class="line">hadoop fs -<span class="built_in">cat</span> &lt;hdfs上的路径&gt;</span><br><span class="line"><span class="comment">#查看文件列表</span></span><br><span class="line">hadoop fs -<span class="built_in">ls</span> /</span><br><span class="line"><span class="comment">#下载文件</span></span><br><span class="line">hadoop fs -get &lt;hdfs上的路径&gt; &lt;linux上文件&gt;</span><br></pre></td></tr></table></figure>
-<p>下一篇：<a href="/hexo_blog/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/" title="在Hadoop基础上Hive的安装">在Hadoop基础上Hive的安装</a></p>
+<h2 id="不重复数">不重复数</h2>
+<p>在2.5亿个正整数中找出不重复的数。</p>
+<h3 id="分治法hashmap">分治法+HashMap</h3>
+<h4 id="说明">说明</h4>
+<p>比如将这些数据平均分为100个批次，每个批次就有250万数。每个批次用循环进行遍历，存入
+<code>HashMap&lt;a, b&gt;</code>，<code>a</code>表示每个数，<code>b</code>表示对应数出现次数，默认为1。每遍历完一批，就对当前
+<code>HashMap</code>进行去重，如果
+<code>b &gt; 1</code>，就排除此数。全部批次进行完筛选之后，剩下的数最后可以直接进行统计，得出不重复的数。</p>
+<h4 id="双间复杂度">双间复杂度</h4>
+<p>时间复杂度：250W * 100轮 + 其他批次。
+这里可以使用多核机器加速运算。</p>
+<p>空间复杂度：int保存数，<code>Key + Value（250W * 4Byte）/ (1024 * 1024)</code>约为
+<code>(Key + 9.5M)</code>内存</p>
+<h3 id="位图法-bitmap">位图法 Bitmap</h3>
+<h4 id="说明-1">说明</h4>
+<p>设计每两个
+<code>bit</code>位，标识一个数的出现情况。如：00表示没有出现，01表示出现一次，10表述出现多次。此题给出的是正整数，所以就是无符号的整型数，占四个字节。</p>
+<p>内存的计算。1B=8b, 4B=32b, 它可以表示的最大整数是<span
+class="math inline">\(2^{32}-1\)</span>, 即需要$2^{32}-1 $约位<span
+class="math inline">\(2^{32}\)</span>个<strong>位</strong>来表示这2.5亿个数，每个状态是两个数，总共就是<span
+class="math inline">\(2^{32}\times 2\)</span>个位。</p>
+<p>一次性申请的位图内存是: $2^{32} $ bit, 即<span
+class="math inline">\(\frac{2^{32}\times2}{(1024\times1024 \times
+8)}=1\)</span> GB。如果加上分治法，甚至都不需要1G内存。</p>
+<p>假设一个数是64, 则对应的bit位是 <span class="math inline">\(64\times
+2=128\)</span>，即127和128共同标识这个数的出现状态（没有出现、出现一次、出现多次）。所以，每当读入了一个数，先找对应的bit位，如果是00就改为01，如果是01就改为10，如果是10就不需要修改。最后扫描整个位图，如果是10，就将下标/2得出出现次数为1的这个数。</p>
+<h2 id="出现次数最多的">出现次数最多的</h2>
+<ol type="1">
+<li>找出一篇文章中，出现次数最多的单词</li>
+<li>10亿个正整数找出重复次数最多的100个整数</li>
+</ol>
+<p>这类问题被叫做Top K问题</p>
+<h3 id="分治法hashmap-1">分治法+HashMap</h3>
+<p>对于文章来说，我们需要规定每批次的字数限制。这里假设100为一批的处理极限，每次处理100个，如果最后一批少于100个就直接当作100来处理。之后每批文字进行切分(<code>split</code>)，比如英文的切割方法可以是空格、标点符号什么的，中文的切割方式可以是空格或者有预先准备好的词库。</p>
+<p>之后就跟上面不重复数字的思路类似了，遍历每批数据，存入
+<code>HashMap&lt;String, Integer&gt;</code>，String表示字符串，Integer对应出现的次数，最大的Integer就是出现次数最多的。</p>
+<p>对数字来说，就是跟不重复数字思路类似，将十亿个数分乘很多份，利用
+<code>HashMap&lt;int, int&gt;</code>来统计，最后找到第二个int最大的那个。每批筛选完之后，就进行排序，选出十亿个数中最大的。</p>
+<h3 id="位图法-bitmap-1">位图法 Bitmap</h3>
+<p>对于文章（字符串）来说，不建议采用</p>
+<p>对于数字来说，此题的背景不需要乘2，将每个数存进位图。出现了一次，bit位=1，没有出现就是0。但是没有办法统计出现次数，所以此题也不适合采用位图法。</p>
+<h2 id="其他">其他</h2>
+<p>在XXX中找出最大、最小、最大的几个、最小的几个。都可以使用分治法+最小堆/最大堆。</p>
 ]]></content>
       <categories>
-        <category>Environment</category>
+        <category>Algorithm</category>
       </categories>
       <tags>
-        <tag>Linux</tag>
-        <tag>Hadoop</tag>
-        <tag>大数据</tag>
+        <tag>HashMap</tag>
+        <tag>位图法</tag>
       </tags>
   </entry>
   <entry>
-    <title>Ubuntu20.04安装深度学习环境及配置个人环境小记</title>
-    <url>/hexo_blog/2021/10/30/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Ubuntu20.04%E5%AE%89%E8%A3%85%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0%E7%8E%AF%E5%A2%83%E5%8F%8A%E9%85%8D%E7%BD%AE%E4%B8%AA%E4%BA%BA%E7%8E%AF%E5%A2%83%E5%B0%8F%E8%AE%B0/</url>
-    <content><![CDATA[<p>Ubuntu20.04安装深度学习环境及配置个人环境小记</p>
+    <title>Mongodb从安装到基本操作</title>
+    <url>/hexo_blog/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/</url>
+    <content><![CDATA[<p>这篇文章主要从零开始介绍如何在Centos7下安装Mongodb,
+用Python3连接Mongodb, 用可视化工具Robo 3T连接Mongodb,
+之后介绍了一些基本的mongo操作方法, 其中含有一些进阶操作</p>
 <span id="more"></span>
-<h3 id="目标配置环境">目标配置环境</h3>
-<p>系统为Ubuntu 20.04,
-显卡为3090，结合其他服务器的环境，准备配置如下环境</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">nvidia-driver-460 <span class="comment"># 英伟达驱动，最低推荐版本为这个</span></span><br><span class="line">CUDA 11.2 (gcc 7) <span class="comment"># 30系显卡应该换为cuda11版本以上，20系显卡可以用cuda10.2版本</span></span><br><span class="line">cudnn 8.6.1</span><br><span class="line"></span><br><span class="line">pytorch 1.10</span><br><span class="line">torchvision 0.11.1</span><br><span class="line">tensorflow 2.5.0</span><br></pre></td></tr></table></figure>
-<h3 id="准备工作">准备工作</h3>
-<p>刚装的系统，啥也没有，所以按照个人习惯先安装一些软件。为方便起见，后文非特殊说明，均用root用户完成安装。可以直接复制下文至sh文件中，更改权限以直接运行。（请保证软件源可正常连接）</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 第一步: 安装一些必备的软件</span></span><br><span class="line">add-apt-repository ppa:graphics-drivers/ppa</span><br><span class="line">apt update -y &amp;&amp; apt upgrade -y</span><br><span class="line">apt install -y git autossh python3-pip vim tmux gpustat tree ranger build-essential gcc-7 g++-7 make curl htop ipython3 zhs</span><br><span class="line"><span class="comment"># gcc与g++降级</span></span><br><span class="line">update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100</span><br><span class="line">update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第二步：安装nvidia驱动</span></span><br><span class="line"><span class="comment"># 禁用nouveau</span></span><br><span class="line"><span class="built_in">chmod</span> 666 /etc/modprobe.d/blacklist.conf</span><br><span class="line"><span class="comment"># 增加相应配置, 若已追加则记得注释下面这一行命令</span></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&quot;</span></span><br><span class="line"><span class="string">blacklist vga16fb</span></span><br><span class="line"><span class="string">blacklist nouveau</span></span><br><span class="line"><span class="string">blacklist rivafb</span></span><br><span class="line"><span class="string">blacklist rivatv</span></span><br><span class="line"><span class="string">blacklist nvidiafb</span></span><br><span class="line"><span class="string">&quot;</span></span><br><span class="line">&gt;&gt; /etc/modprobe.d/blacklist.conf</span><br><span class="line"><span class="comment"># 更新配置</span></span><br><span class="line">update-initramfs -u</span><br><span class="line">apt install nvidia-driver-460 -y <span class="comment"># 英伟达460驱动，按需更改</span></span><br></pre></td></tr></table></figure>
-<p>执行完成之后，重启服务器。使用<code>nvidia-smi</code>查看gpu是否正常，或使用安装好的<code>gpustat</code></p>
-<h3 id="cuda及cudnn安装">cuda及cudnn安装</h3>
-<h4 id="下载">下载</h4>
-<p>这里需要从官网下载安装包，cuda安装包可以直接通过命令下载。cudnn需要注册英伟达开发者账号，下载四个文件</p>
+<h2 id="安装mongodbcentos7">安装Mongodb(Centos7)</h2>
+<ol type="1">
+<li>官网上找到需要下载的版本压缩包进行下载，<a
+href="https://fastdl.mongodb.org/linux/">官网链接</a>。或者在命令行下使用<code>wget</code>命令进行下载</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 这里下载的是3.4.10版本</span></span><br><span class="line">wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel70-3.4.10.tgz</span><br></pre></td></tr></table></figure>
+<ol start="2" type="1">
+<li>进行解压缩</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxvf mongodb-linux-x86_64*.tgz</span><br><span class="line"><span class="comment"># 重命名</span></span><br><span class="line"><span class="built_in">mv</span> mongodb-linux-x86_64* mongodb</span><br><span class="line"><span class="comment"># 移动到你想放置的目录下，这里我放在/opt目录下</span></span><br><span class="line"><span class="built_in">mv</span> mongodb /opt/</span><br></pre></td></tr></table></figure>
+<ol start="3" type="1">
+<li>进行相应的配置，mongodb默认没有任何配置</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb主目录</span></span><br><span class="line"><span class="built_in">cd</span> /opt/</span><br><span class="line"><span class="comment"># 建立存放数据文件和日志文件的目录</span></span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/logs</span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/db</span><br><span class="line"><span class="comment"># 创建配置文件，并写入如下配置</span></span><br><span class="line">vim bin/mongodb.conf</span><br><span class="line">  `</span><br><span class="line">    <span class="comment"># 设置数据文件的存放目录</span></span><br><span class="line">    dbpath = /opt/mongodb/data/test/db</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置日志文件的存放目录及其日志文件名</span></span><br><span class="line">    logpath = /opt/mongodb/data/test/logs/mongodb.log</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置端口号（默认的端口号是27017，可以根据个人需求进行更改）</span></span><br><span class="line">    port = 27017</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 设置为以守护进程的方式运行，即在后台运行</span></span><br><span class="line">    fork = <span class="literal">true</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 是否不允许表扫描</span></span><br><span class="line">    nohttpinterface = <span class="literal">true</span></span><br><span class="line">  `</span><br><span class="line"><span class="comment"># 保存退出</span></span><br></pre></td></tr></table></figure>
+<ol start="4" type="1">
+<li>启动mongodb</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 以配置文件的方式启动</span></span><br><span class="line">./bin/mongod --config mongodb.conf</span><br></pre></td></tr></table></figure>
+<p>报错一：
+<code>ERROR: child process failed, exited with error number 1</code>
+检查mongodb.conf的文件路径是否配置错误</p>
+<p>报错二：<code>ERROR: child process failed, exited with error number 100</code>
+很可能是没有正常关闭导致的，那么可以删除
+<code>mongod.lock</code>文件，这里对应我的配置路径在<code>data/</code>里面</p>
+<ol start="5" type="1">
+<li>其他</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 链接命令，方便调用mongo命令</span></span><br><span class="line"><span class="built_in">ln</span> -s /opt/mongodb/bin/mongo /usr/bin</span><br><span class="line"><span class="comment"># 查看mongodb进程</span></span><br><span class="line">ps -aux |grep mongodb</span><br><span class="line"><span class="comment"># 检查端口运行情况</span></span><br><span class="line">netstat -lanp | grep 27017</span><br><span class="line"><span class="comment"># 终止mongodb服务，PID从ps命令获取</span></span><br><span class="line"><span class="built_in">kill</span> -15 PID</span><br><span class="line"><span class="comment"># 添加自启动命令</span></span><br><span class="line">vim /etc/rc.local</span><br><span class="line"><span class="comment"># 末尾追加一行</span></span><br><span class="line">`/opt/mongodb/bin/mongod --config mongodb.conf`</span><br><span class="line"><span class="comment"># 保存退出</span></span><br></pre></td></tr></table></figure>
+<ol start="6" type="1">
+<li>设置密码权限，默认无密码</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb的交互环境</span></span><br><span class="line">./mongo</span><br><span class="line"><span class="comment"># 如果进入失败，请检查是否添加了软链接和是否启动了mongodb服务</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行admin数据库，创建管理员用户root，密码为password，权限是超级用户（最高）</span></span><br><span class="line">&gt;use admin</span><br><span class="line">&gt;db.createUser(&#123;user:<span class="string">&quot;root&quot;</span>,<span class="built_in">pwd</span>:<span class="string">&quot;password&quot;</span>,roles:[<span class="string">&quot;root&quot;</span>]&#125;)</span><br><span class="line"><span class="comment"># 验证是否创建成功，返回1表示成功</span></span><br><span class="line">&gt;db.auth(&#123;<span class="string">&quot;root&quot;</span>, <span class="string">&quot;password&quot;</span>&#125;)</span><br><span class="line"><span class="comment"># 退出交互环境</span></span><br><span class="line">&gt;<span class="built_in">exit</span></span><br><span class="line"><span class="comment"># 重启mongodb服务</span></span><br><span class="line"><span class="comment"># 杀死mongodb进程，参照上面的方法</span></span><br><span class="line"><span class="comment"># 以密码权限验证启动服务</span></span><br><span class="line">opt/mongodb/mongod --config mongodb.conf --auth</span><br></pre></td></tr></table></figure>
+<h2 id="使用python连接mongodb">使用Python连接Mongodb</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载第三方包</span></span><br><span class="line">pip3 install pymongo</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># /usr/bin/python3</span></span><br><span class="line"><span class="keyword">from</span> pymongo <span class="keyword">import</span> MongoClient</span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法一</span></span><br><span class="line"><span class="comment"># host主机名，27017连接端口</span></span><br><span class="line">client = MongoClient(host, <span class="number">27017</span>)</span><br><span class="line">db_auth = client.admin</span><br><span class="line"><span class="comment"># 登陆的用户名(username)和密码(password)</span></span><br><span class="line">db_auth.authenticate(username, password)</span><br><span class="line"><span class="comment"># 连接指定数据库，数据库名为db_name</span></span><br><span class="line">db_name = client[<span class="string">&quot;db_name&quot;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line"><span class="comment"># 用户名(username)、密码(password)、主机名(host)、端口(port)。注：这里的password不能出现@符号，如果用@符号就需要使用方法一</span></span><br><span class="line">client = MongoClient(<span class="string">&quot;mongodb://username:password@host:port&quot;</span>)</span><br><span class="line"><span class="comment"># 连接指定数据库，数据库名为db_name</span></span><br><span class="line">db_name = client[<span class="string">&quot;db_name&quot;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 操作数据库，db_name[&quot;db_set&quot;]=db_name.db_set</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">查找数据</span></span><br><span class="line"><span class="string">在db_name数据库中的db_set集合里面找到name是a_name的json</span></span><br><span class="line"><span class="string">find是返回一个指针（可以使用列表的方式来读取），find_one返回一个dict</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line">db_name[<span class="string">&quot;db_set&quot;</span>].find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;a_name&quot;</span>&#125;)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 插入数据</span></span><br><span class="line"><span class="comment"># 插入一个json到set1集合中，a_dict是一个字典（如果set1不存在则会被自动创建）</span></span><br><span class="line">db_name.set1.insert_one(a_dict)</span><br><span class="line"><span class="comment"># 插入多个json到set1集合中，a_dict_lst是一个列表，里面的元素是由字典组成的</span></span><br><span class="line">db_name.set1.insert_many(a_dict_lst)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 更新数据</span></span><br><span class="line"><span class="comment"># 查找到一个数据之后，对里面的某个数据进行更改，再使用save方法保存</span></span><br><span class="line">data_one = db_name[<span class="string">&quot;db_set&quot;</span>].find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;a_name&quot;</span>&#125;)</span><br><span class="line">data_one[<span class="string">&#x27;age&#x27;</span>] += <span class="number">3</span></span><br><span class="line">db_name.db_set.save(data_one)</span><br><span class="line"><span class="comment"># 更新多条记录，找到db_set集合中满足age=20，sex=0的数据，将name改为user1</span></span><br><span class="line">db_name.db_set.update(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;, &#123;<span class="string">&quot;$set&quot;</span>:&#123;<span class="string">&quot;age&quot;</span>: <span class="number">20</span>, <span class="string">&quot;sex&quot;</span>: <span class="number">0</span>&#125;&#125;)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除数据</span></span><br><span class="line"><span class="built_in">id</span> = db_name.db_set.find_one(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;)[<span class="string">&quot;_id&quot;</span>]</span><br><span class="line"><span class="comment"># 根据 id 删除一条记录</span></span><br><span class="line">db_name.db_set.remove(<span class="built_in">id</span>)</span><br><span class="line"><span class="comment"># 删除集合里的所有记录</span></span><br><span class="line">db_name.db_set.remove()</span><br><span class="line"><span class="comment"># 删除name=user1的记录</span></span><br><span class="line">db_name.db_set.remove(&#123;<span class="string">&quot;name&quot;</span>: <span class="string">&quot;user1&quot;</span>&#125;)</span><br></pre></td></tr></table></figure>
+<h2 id="使用robo-3t连接mongodb">使用Robo 3t连接Mongodb</h2>
+<p>强烈推荐Robo 3t，<a href="https://robomongo.org/">官网地址</a></p>
+<ol type="1">
+<li><p>到官网找到对应的系统版本进行下载</p></li>
+<li><p>安装</p>
 <ul>
-<li><p><code>cudnn-11.2-linux-x64-v8.1.0.77</code></p></li>
-<li><p><code>libcudnn8_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
-<li><p><code>libcudnn8-dev_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
-<li><p><code>libcudnn8-samples_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
+<li><p>Windows的安装十分简单就不赘述了</p></li>
+<li><p>Ubuntu的安装 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxzf robomongo*.tar.gz</span><br><span class="line"><span class="comment"># 修改文件名（可选）</span></span><br><span class="line"><span class="built_in">mv</span> robomongo* robo3t</span><br><span class="line"><span class="comment"># 移动到/opt目录（可选）</span></span><br><span class="line"><span class="built_in">mv</span> robo3t /opt</span><br><span class="line"><span class="comment"># 建立软链接</span></span><br><span class="line"><span class="built_in">ln</span> -s /opt/robo3t/robomongo /usr/bin</span><br><span class="line"><span class="comment"># 启动</span></span><br><span class="line">robomongo</span><br></pre></td></tr></table></figure> <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 如果报如下错误</span></span><br><span class="line">`</span><br><span class="line">This application failed to start because it could not find or load the Qt platform plugin <span class="string">&quot;xcb&quot;</span> <span class="keyword">in</span> <span class="string">&quot;&quot;</span>.</span><br><span class="line"></span><br><span class="line">Available platform plugins are: xcb.</span><br><span class="line"></span><br><span class="line">Reinstalling the application may fix this problem.</span><br><span class="line">Aborted (core dumped)</span><br><span class="line">`</span><br><span class="line"><span class="comment"># 解决办法</span></span><br><span class="line"><span class="built_in">mkdir</span> ~/robo-backup</span><br><span class="line"><span class="built_in">mv</span> /opt/robo3t/lib/libstdc++* ~/robo-backup/</span><br><span class="line"><span class="comment"># 之后便可以重新启动mongo试试看了</span></span><br></pre></td></tr></table></figure></p></li>
+</ul></li>
+<li><p>使用</p>
+<ol type="1">
+<li>首先创建新的连接，如下图所示。在<code>Connection中，``Name</code>为自定义连接名，<code>Address</code>填入对应的<code>host</code>和<code>port</code>；
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/mongodb/20171124171027733.png" /></li>
+<li>在<code>Authentication</code>中，勾选<code>Perform authentication</code>，填上对应的<code>User Name</code>和<code>Password</code>。填写完成之后可以点击<code>Test</code>验证是否能够连接成功，如果无误的话，可以点击<code>Save</code>保存退出。</li>
+</ol>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/mongodb/20171124171111807.png" /></p>
+<ol start="3" type="1">
+<li>连接Mongodb。点击<code>Connect</code>，之后就可以可视化数据库中的数据。也可以使用鼠标简单的进行创建/删除数据库，新建/修改信息，也可以使用命令查找数据。</li>
+</ol></li>
+</ol>
+<h2 id="curd操作主要是查找数据">CURD操作（主要是查找数据）</h2>
+<p><strong>CRUD</strong></p>
+<ul>
+<li>C: Create Operations, 创建</li>
+<li>R: Read Operations, 读取</li>
+<li>U: Update Operations, 更新</li>
+<li>D: Delete Operations, 删除</li>
 </ul>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget https://developer.download.nvidia.com/compute/cuda/11.2.2/local_installers/cuda_11.2.2_460.32.03_linux.run &amp;&amp; <span class="built_in">chmod</span> 755 cuda_11.2.2_460.32.03_linux.run</span><br><span class="line"></span><br><span class="line">./cuda_11.2.2_460.32.03_linux.run</span><br></pre></td></tr></table></figure>
-<p>下载完直接运行，安装过程中需要把Drivers这项取消，因为自带了显卡驱动。</p>
-<h4 id="环境变量">环境变量</h4>
-<p>在/etc/bash.bashrc或/etc/profile前加入如下两行。如果profile不行就换bashrc。一般是编辑<code>vim ~/.bashrc</code>下，但为保证所有用户都生效，所以需要在<code>/etc</code>目录下编辑</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">export</span> PATH=/usr/local/cuda-11.2/bin<span class="variable">$&#123;PATH:+:<span class="variable">$&#123;PATH&#125;</span>&#125;</span></span><br><span class="line"><span class="built_in">export</span> LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64<span class="variable">$&#123;LD_LIBRARY_PATH:+:<span class="variable">$&#123;LD_LIBRARY_PATH&#125;</span>&#125;</span></span><br></pre></td></tr></table></figure>
-<p>至此，cuda安装完成</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 检测是否安装成功</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/cuda/samples/1_Utilities/deviceQuery</span><br><span class="line">make clean &amp;&amp; make -j4 &amp;&amp; ./deviceQuery</span><br><span class="line"><span class="comment"># 最后一行输出，Result = PASS表示成功</span></span><br></pre></td></tr></table></figure>
-<h4 id="cudnn配置">cudnn配置</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tar -zxvf cudnn-11.2-linux-x64-v8.1.0.77.tgz </span><br><span class="line"><span class="built_in">cp</span> cuda/include/cudnn.h /usr/local/cuda/include </span><br><span class="line"><span class="built_in">cp</span> cuda/lib64/libcudnn* /usr/local/cuda/lib64  </span><br><span class="line"><span class="built_in">chmod</span> a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn*  </span><br><span class="line"></span><br><span class="line">dpkg -i libcudnn8_8.1.0.77-1+cuda11.2_amd64.deb      </span><br><span class="line">dpkg -i libcudnn8-dev_8.1.0.77-1+cuda11.2_amd64.deb  </span><br><span class="line">dpkg -i libcudnn8-samples_8.1.0.77-1+cuda11.2_amd64.deb </span><br><span class="line"></span><br><span class="line"><span class="comment"># cudnn检测，更高版本的cudnn需要把cudnn.h换成cudnn_version.h</span></span><br><span class="line"><span class="built_in">cat</span> /usr/local/cuda/include/cudnn.h | grep CUDNN_MAJOR -A 2</span><br><span class="line"><span class="comment"># 利用程序检查，可能需要重启</span></span><br><span class="line"><span class="built_in">cd</span> /usr/src/cudnn_samples_v7/conv_sample</span><br><span class="line">make clean &amp;&amp; make -j4 &amp;&amp; ./conv_sample</span><br><span class="line"><span class="comment"># 最后一行输出，Test passed！表示成功</span></span><br></pre></td></tr></table></figure>
-<p>通过<code>ldconfig</code>可检查动态链接库是否链接正确。若有输出，可以使用<code>ln -sf</code>重新链接。</p>
-<h3 id="python库安装">Python库安装</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">pip3 install tensorflow-gpu==2.5.0 -i https://pypi.tuna.tsinghua.edu.cn/simple</span><br><span class="line">pip3 install torch==1.10.0+cu113 torchvision==0.11.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html</span><br><span class="line">pip3 install black tqdm flask sklearn</span><br></pre></td></tr></table></figure>
-<h4 id="检查是否安装成功">检查是否安装成功</h4>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.cuda.is_available()) </span><br><span class="line"><span class="built_in">print</span>(torch.backends.cudnn.is_available()) </span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.__version__)</span><br><span class="line"><span class="built_in">print</span>(torch.version.cuda)</span><br><span class="line"><span class="built_in">print</span>(torch.backends.cudnn.version())</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> tensorflow <span class="keyword">as</span> tf</span><br><span class="line">gpu_device_name = tf.test.gpu_device_name()</span><br><span class="line"><span class="built_in">print</span>(gpu_device_name)   </span><br><span class="line">tf.test.is_gpu_available() <span class="comment"># 输出True</span></span><br></pre></td></tr></table></figure>
-<p>若tf未检测到GPU，可以看输出过程中的哪些动态库文件加载失败，手动配置一下。</p>
-<p>至此，深度学习安装环境配置完成。</p>
-<h3 id="个人环境">个人环境</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建用户</span></span><br><span class="line">adduser xxx</span><br><span class="line"><span class="comment"># 输入配置的密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 增加root权限</span></span><br><span class="line"><span class="built_in">chmod</span> u+w /etc/sudoers</span><br><span class="line">vim /etc/sudoers</span><br><span class="line">​```</span><br><span class="line">root    ALL=(ALL:ALL) ALL</span><br><span class="line"><span class="comment"># 增加一行</span></span><br><span class="line">xxx    ALL=(ALL:ALL) ALL</span><br><span class="line">​```</span><br><span class="line"><span class="built_in">chmod</span> u-w /etc/sudoers</span><br></pre></td></tr></table></figure>
-<h4 id="ssh文件配置">ssh文件配置</h4>
-<p>方便起见直接把其他服务器的<code>.ssh</code>文件夹下的<code>id_rsa</code>，<code>id_rsa.pub</code>和<code>authorized_keys</code>复制过来。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 复制完成之后，修改对应权限才能使用</span></span><br><span class="line"><span class="built_in">chmod</span> 755 ~/.ssh/  </span><br><span class="line"><span class="built_in">chmod</span> 600 ~/.ssh/id_rsa ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys</span><br><span class="line"><span class="built_in">chmod</span> 644 ~/.ssh/known_hosts</span><br></pre></td></tr></table></figure>
-<h4 id="oh-my-zsh">oh-my-zsh</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载并安装</span></span><br><span class="line">wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | sh</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装插件</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/zsh-users/zsh-autosuggestions <span class="variable">$&#123;ZSH_CUSTOM:-~/.oh-my-zsh/custom&#125;</span>/plugins/zsh-autosuggestions</span><br><span class="line">git <span class="built_in">clone</span> https://github.com/zsh-users/zsh-syntax-highlighting.git <span class="variable">$&#123;ZSH_CUSTOM:-~/.oh-my-zsh/custom&#125;</span>/plugins/zsh-syntax-highlighting</span><br><span class="line"></span><br><span class="line"><span class="comment"># vim ~/.zshrc, 最后一行加入如下配置</span></span><br><span class="line">plugins=(</span><br><span class="line">        git</span><br><span class="line">        zsh-autosuggestions</span><br><span class="line">        zsh-syntax-highlighting</span><br><span class="line">)</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/custom/plugins/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/custom/plugins/zsh-autosuggestions/zsh-autosuggestions.zsh</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/oh-my-zsh.sh</span><br></pre></td></tr></table></figure>
-<h4 id="配置vim">配置vim</h4>
-<p>编辑<code>.vimrc</code></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">set</span> number</span><br><span class="line"></span><br><span class="line">syntax on</span><br><span class="line">colorscheme darkblue</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">syntax <span class="built_in">enable</span></span><br><span class="line">filetype plugin on</span><br><span class="line"><span class="built_in">let</span> g:go_disable_autoinstall = 0</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_functions = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_methods = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_structs = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_operators = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_build_constraints = 1</span><br><span class="line"><span class="string">&quot; &quot;</span> syntax on</span><br><span class="line"><span class="string">&quot;&quot;</span> filetype plugin indent on</span><br><span class="line"><span class="built_in">let</span> g:pymode_python = <span class="string">&#x27;python3&#x27;</span></span><br><span class="line"><span class="built_in">set</span> number</span><br><span class="line"><span class="built_in">set</span> cin</span><br><span class="line"><span class="built_in">set</span> sw=8</span><br><span class="line"><span class="built_in">set</span> ts=8</span><br><span class="line"><span class="built_in">set</span> sm</span><br><span class="line">colorscheme darkblue</span><br><span class="line"><span class="built_in">set</span> viminfo=<span class="string">&#x27;10,\&quot;100,:20,%,n~/.viminfo </span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">au BufReadPost * if line(&quot;&#x27;</span>\&quot;<span class="string">&quot;) &gt; 0|if line(&quot;</span><span class="string">&#x27;\&quot;&quot;) &lt;= line(&quot;$&quot;)|exe(&quot;norm &#x27;</span>\&quot;<span class="string">&quot;)|else|exe &quot;</span>norm $<span class="string">&quot;|endif|endif</span></span><br></pre></td></tr></table></figure>
-<h4 id="远程连接">远程连接</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">autossh -gfCNR 远程端口:localhost:22（默认本地ssh端口） 公网用户名@公网IP</span><br></pre></td></tr></table></figure>
-<p>需要提前配好免密登录，远程端口在公网服务器上需要开放防火墙。</p>
-<p>远程连接登录<code>ssh 服务器用户@公网IP -p 远程端口</code></p>
+<p>mongo数据库至上而下是：<code>Database</code>--&gt;<code>Collections</code>--&gt;<code>Documents</code>。我在这里分别理解成数据库、集合、记录。</p>
+<p>这里直接用官方给的数据来操作。假设以下数据放在了<code>fruits</code>集合中。</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="number">5</span><span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">-1</span><span class="punctuation">,</span> <span class="number">3</span><span class="punctuation">]</span><span class="punctuation">,</span> <span class="attr">&quot;links&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="punctuation">&#123;</span><span class="attr">&quot;Uin&quot;</span><span class="punctuation">:</span> <span class="number">124</span><span class="punctuation">,</span> <span class="attr">&quot;NickName&quot;</span><span class="punctuation">:</span> <span class="string">&quot;123&quot;</span><span class="punctuation">&#125;</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;bananas&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="number">7</span><span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">1</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;oranges&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="punctuation">&#123;</span> <span class="attr">&quot;in stock&quot;</span><span class="punctuation">:</span> <span class="number">8</span><span class="punctuation">,</span> <span class="attr">&quot;ordered&quot;</span><span class="punctuation">:</span> <span class="number">12</span> <span class="punctuation">&#125;</span><span class="punctuation">,</span>  <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">5</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span> <span class="punctuation">&#125;</span></span><br><span class="line"><span class="punctuation">&#123;</span> <span class="attr">&quot;_id&quot;</span><span class="punctuation">:</span> <span class="string">&quot;avocados&quot;</span><span class="punctuation">,</span> <span class="attr">&quot;qty&quot;</span><span class="punctuation">:</span> <span class="string">&quot;fourteen&quot;</span> <span class="punctuation">,</span> <span class="attr">&quot;score&quot;</span><span class="punctuation">:</span> <span class="punctuation">[</span><span class="number">5</span><span class="punctuation">,</span> <span class="number">5</span><span class="punctuation">]</span><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
+<h3 id="基本操作">基本操作</h3>
+<p><code>json</code>数据的嵌套查找，就是说，如果有字段包含json，可以直接用<code>.</code>进行连接。</p>
+<p>查找<code>qty</code>中的<code>ordered</code>为12的字段</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty.ordered&quot;: <span class="number">12</span>&#125;)</span><br></pre></td></tr></table></figure>
+<p>多重筛选，同时满足多个要求的记录</p>
+<p>查找<code>qty</code>为5，且<code>_id</code>为"apples"的记录</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>, &quot;_id&quot;: &quot;apples&quot;&#125;)</span><br></pre></td></tr></table></figure>
+<p>第一个大括号内容表示筛选条件，第二个大括号表示显示内容</p>
+<p>筛选条件就不说明了，第二个大括号表示打印<code>_id</code>字段和<code>score</code>字段，不打印其他字段，</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>&#125;, &#123; &quot;_id&quot;: <span class="number">1</span> , &quot;score&quot;: <span class="number">1</span>&#125;)</span><br></pre></td></tr></table></figure>
+<h3 id="比较">比较</h3>
+<p>下面至上而下分别表示，在<code>fruits</code>集合中查找字段<code>qty</code>与4的大小关系所有记录。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">4</span>&#125;)          # 等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gt&quot;: <span class="number">4</span>&#125;&#125;)   # 大于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$lt&quot;: <span class="number">4</span>&#125;&#125;)   # 小于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gte&quot;: <span class="number">4</span>&#125;&#125;)  # 大于等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$lte&quot;: <span class="number">4</span>&#125;&#125;)  # 小于等于</span><br><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$ne&quot;: <span class="number">4</span>&#125;&#125;)   # 不等于</span><br></pre></td></tr></table></figure>
+<p>当然，也可以两个比较符一起操作，比如下面这样。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: &#123;&quot;$gt&quot;: <span class="number">3</span>, &quot;$lt&quot;: <span class="number">5</span>&#125;&#125;)  # 大于<span class="number">3</span>小于<span class="number">5</span>。</span><br></pre></td></tr></table></figure>
+<p>如果对<code>score</code>这个字段使用比较，比如下面这样。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;score&quot;: &#123;&quot;$gt&quot;: <span class="number">2</span>, &quot;$lt&quot;: <span class="number">5</span>&#125;&#125;)  # 大于<span class="number">3</span>小于<span class="number">5</span>。</span><br></pre></td></tr></table></figure>
+<p>将会返回下面两条记录。理由是，字段里的数组里有含有大于2小于5的数字。</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">/* 1 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;qty&quot;</span> <span class="punctuation">:</span> <span class="number">5</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">-1</span><span class="punctuation">,</span></span><br><span class="line">        <span class="number">3</span></span><br><span class="line">    <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;links&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="punctuation">&#123;</span></span><br><span class="line">            <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> <span class="number">124</span><span class="punctuation">,</span></span><br><span class="line">            <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;123&quot;</span></span><br><span class="line">        <span class="punctuation">&#125;</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br><span class="line"></span><br><span class="line"><span class="comment">/* 2 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;bananas&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;qty&quot;</span> <span class="punctuation">:</span> <span class="number">7</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">1</span><span class="punctuation">,</span></span><br><span class="line">        <span class="number">5</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
+<h3 id="关系">关系</h3>
+<h4 id="in">$in</h4>
+<p>查找<code>_id</code>字段中包含<code>5</code>或者<code>ObjectId("507c35dd8fada716c89d0013")</code>的记录</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;_id&quot;: &#123;&quot;$in&quot;:[<span class="number">5</span>, ObjectId(&quot;507c35dd8fada716c89d0013&quot;)] &#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="elemmatch">$elemMatch</h4>
+<p>用于字段中含有数组，且数组中含有<code>json</code>数据的的查找。</p>
+<p>查找<code>links</code>这个含有数组的字段，找出数组中<code>Uin</code>等于124的记录</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;links&quot;: &#123;&quot;$elemMatch&quot;: &#123;&quot;Uin&quot;: <span class="number">124</span>&#125;&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="slice">$slice</h4>
+<p>切片，只打印字段中数组前n个数据</p>
+<p>筛选出记录之后，只打印记录<code>score</code>字段的前1条数据</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;qty&quot;: <span class="number">5</span>&#125;, &#123;&quot;_id&quot;: <span class="number">1</span>, &quot;score&quot;: &#123;&quot;$slice&quot;: <span class="number">1</span>&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<p>输出如下</p>
+<figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">/* 1 */</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">    <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;apples&quot;</span><span class="punctuation">,</span></span><br><span class="line">    <span class="attr">&quot;score&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">        <span class="number">-1</span></span><br><span class="line">    <span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure>
+<h4 id="regex">$regex</h4>
+<p>使用正则表达式，更多操作请查看官方文档: <a
+href="https://docs.mongodb.com/manual/reference/operator/query/regex/index.html">$regex</a></p>
+<p>找出<code>_id</code>字段中所有以a开头的记录</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find(&#123;&quot;_id&quot;: &#123;&quot;$regex&quot;: &quot;^a&quot;&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h3 id="后缀">后缀</h3>
+<h4 id="sort排序">sort——排序</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().sort(&#123;&quot;qty&quot;: <span class="number">1</span>&#125;) # 升序</span><br><span class="line">db.fruits.find().sort(&#123;&quot;qty&quot;: <span class="number">-1</span>&#125;) # 降序</span><br></pre></td></tr></table></figure>
+<h4 id="limit限制">limit——限制</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().limit(<span class="number">3</span>) # 对查询结果只输出前<span class="number">3</span>个，可以加在sort后面结合使用</span><br></pre></td></tr></table></figure>
+<h4 id="skip跳过">skip——跳过</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="keyword">skip</span>(<span class="number">3</span>) # 对查询结果跳过输出前<span class="number">3</span>个，即不输出前<span class="number">3</span>个，可以加在sort后面结合使用</span><br></pre></td></tr></table></figure>
+<h4 id="count计数">count——计数</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="built_in">count</span>()  # 返回查询结果总数</span><br></pre></td></tr></table></figure>
+<h4
+id="collation基于特定语言进行操作">collation——基于特定语言进行操作</h4>
+<p>没有怎么看懂，我理解的大概意思就是说，运行根据特定语言来进行查找。感觉用的不多，这里贴上官方给的例子</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.fruits.find().<span class="keyword">collation</span>( &#123; locale: &quot;en_US&quot;, strength: <span class="number">1</span> &#125; )</span><br></pre></td></tr></table></figure>
+<h3 id="个人常用操作">个人常用操作</h3>
+<p>例子： <figure class="highlight json"><table><tr><td class="code"><pre><span class="line"><span class="comment">// test_db</span></span><br><span class="line"><span class="punctuation">&#123;</span></span><br><span class="line">   <span class="attr">&quot;_id&quot;</span> <span class="punctuation">:</span> ObjectId(<span class="string">&quot;56063f17ade2f21f36b03133&quot;</span>)<span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;title&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;MongoDB 教程&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;description&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;MongoDB 是一个 Nosql 数据库&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;by&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;菜鸟教程&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;url&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;http://www.runoob.com&quot;</span><span class="punctuation">,</span></span><br><span class="line">   <span class="attr">&quot;tags&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span></span><br><span class="line">           <span class="string">&quot;mongodb&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="string">&quot;database&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="string">&quot;NoSQL&quot;</span></span><br><span class="line">   <span class="punctuation">]</span><span class="punctuation">,</span></span><br><span class="line">     <span class="attr">&quot;links&quot;</span> <span class="punctuation">:</span> <span class="punctuation">[</span><span class="punctuation">&#123;</span></span><br><span class="line">           <span class="attr">&quot;UserName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;@a1bf7a55ad8478017b95124774687769&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> <span class="number">12266535</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;一二三&quot;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#125;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#123;</span></span><br><span class="line">           <span class="attr">&quot;UserName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;@5571e120b5d8f54f93338c76f62b9c419b818c29373dca9f331c2911a203bfbe&quot;</span><span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;Uin&quot;</span> <span class="punctuation">:</span> NumberLong(<span class="number">3379331828</span>)<span class="punctuation">,</span></span><br><span class="line">           <span class="attr">&quot;NickName&quot;</span> <span class="punctuation">:</span> <span class="string">&quot;灵灵八&quot;</span><span class="punctuation">,</span></span><br><span class="line">       <span class="punctuation">&#125;</span><span class="punctuation">]</span></span><br><span class="line"><span class="punctuation">&#125;</span></span><br></pre></td></tr></table></figure></p>
+<h4 id="返回查找结果的数目">返回查找结果的数目</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;title&quot;: &quot;MongoDB 教程&quot;&#125;).<span class="built_in">count</span>()</span><br></pre></td></tr></table></figure>
+<h4
+id="查找某个key中的数组是否某包含某内容">查找某个key中的数组是否某包含某内容</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;tags&quot;: &#123;&quot;$in&quot;: [&quot;mongodb&quot;]&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="使用正则进行查找">使用正则进行查找</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;url&quot;: &#123;&quot;$regex&quot;: &quot;^http&quot;&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h4 id="查找某个由数组构成的key方法">查找某个由数组构成的key方法</h4>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">db.test_db.find(&#123;&quot;links&quot;:&#123;&quot;$elemMatch&quot;:&#123;&quot;Uin&quot;:<span class="number">12266535</span>,&quot;NickName&quot;:&quot;一二三&quot;&#125;&#125;&#125;)</span><br></pre></td></tr></table></figure>
+<h2 id="附录">附录</h2>
+<h3 id="数据库权限说明表">数据库权限说明表</h3>
+<table>
+<colgroup>
+<col style="width: 23%" />
+<col style="width: 76%" />
+</colgroup>
+<thead>
+<tr>
+<th>名称</th>
+<th style="text-align: left;">权限</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>数据库用户角色</td>
+<td style="text-align: left;">read、readWrite</td>
+</tr>
+<tr>
+<td>数据库管理角色</td>
+<td style="text-align: left;">dbAdmin、dbOwner、userAdmin</td>
+</tr>
+<tr>
+<td>集群管理角色</td>
+<td
+style="text-align: left;">clusterAdmin、clusterManager、clusterMonitor、hostManager</td>
+</tr>
+<tr>
+<td>备份恢复角色</td>
+<td style="text-align: left;">backup、restore</td>
+</tr>
+<tr>
+<td>所有数据库角色</td>
+<td
+style="text-align: left;">readAnyDatabase、readWriteAnyDatabase、userAdminAnyDatabase、dbAdminAnyDatabase</td>
+</tr>
+<tr>
+<td>超级用户角色</td>
+<td style="text-align: left;">root</td>
+</tr>
+<tr>
+<td>提供了系统超级用户的访问</td>
+<td style="text-align: left;">dbOwner
+、userAdmin、userAdminAnyDatabase</td>
+</tr>
+<tr>
+<td>内部角色</td>
+<td style="text-align: left;">__system</td>
+</tr>
+</tbody>
+</table>
+<p>下一篇：<a href="/hexo_blog/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/" title="Mongodb：副本集配置">Mongodb：副本集配置</a></p>
 ]]></content>
       <categories>
-        <category>Linux</category>
+        <category>Database</category>
       </categories>
       <tags>
-        <tag>CUDA</tag>
-        <tag>PyTorch</tag>
-        <tag>Ubuntu</tag>
-        <tag>Tensorflow</tag>
-        <tag>CUDNN</tag>
+        <tag>mongodb</tag>
+        <tag>数据库</tag>
       </tags>
   </entry>
   <entry>
-    <title>在Hadoop基础上Hive的安装</title>
-    <url>/hexo_blog/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了Hive在Hadoop上的安装及一些基本的Hive操作</p>
+    <title>Leetcode 字符串刷题记录</title>
+    <url>/hexo_blog/2023/08/28/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E5%AD%97%E7%AC%A6%E4%B8%B2/</url>
+    <content><![CDATA[<p>字符串刷题分类</p>
 <span id="more"></span>
-<p>写在开头 环境 Hadoop单机 Centos7 Hadoop-2.7.3
-hadoop位置：/usr/loacl/hadoop</p>
-<p>参考文章 <a
-href="http://yanliu.org/2015/08/13/Hadoop%E9%9B%86%E7%BE%A4%E4%B9%8BHive%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">Hadoop集群之Hive安装配置</a></p>
-<h2 id="下载hive">下载Hive</h2>
-<p>下载源码包 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#在hadoop目录下操作</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop</span><br><span class="line"></span><br><span class="line"><span class="comment">#用wget下载</span></span><br><span class="line">wget http://mirrors.cnnic.cn/apache/hive/hive-1.2.1/apache-hive-1.2.1-bin.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#也可以在图形界面下下载之后，上传压缩包</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#解压缩包</span></span><br><span class="line">tar -zxvf apache-hive-1.2.1-bin.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#配置环境变量</span></span><br><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#位置在之前配置的变量之后，大概12行左右，因为之前配置了jdk和hadoop变量</span></span><br><span class="line"><span class="built_in">export</span> HIVE_HOME=/usr/local/hadoop/apache-hive-1.2.2-bin</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$HIVE_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"><span class="comment">#适当位置添加&quot;$HIVE_HOME/bin:&quot;</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#使文件生效</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure></p>
-<h2 id="安装mysql">安装MySQL</h2>
-<p>MariaDB数据库管理系统是MySQL的一个分支，主要由开源社区在维护，采用GPL授权许可。开发这个分支的原因之一是：甲骨文公司收购了MySQL后，有将MySQL闭源的潜在风险，因此社区采用分支的方式来避开这个风险。MariaDB的目的是完全兼容MySQL，包括API和命令行，使之能轻松成为MySQL的代替品。</p>
-<p>在Centos7下，使用命令安装mysql，会安装成mariadb。</p>
-<p>在这里先介绍安装MariaDB，原因如上。</p>
-<h3 id="安装mariadb">安装MariaDB</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#安装</span></span><br><span class="line">yum install mariadb-server mariadb</span><br><span class="line"></span><br><span class="line"><span class="comment">#启动</span></span><br><span class="line">systemctl start mariadb</span><br><span class="line"></span><br><span class="line"><span class="comment">#进入MySQL</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#命令行变成如下，可能有点不习惯</span></span><br><span class="line">MariaDB [(none)]&gt;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#P.S.</span></span><br><span class="line"><span class="comment">#相关命令</span></span><br><span class="line">systemctl start mariadb <span class="comment">#启动MariaDB</span></span><br><span class="line">systemctl stop mariadb <span class="comment">#停止MariaDB</span></span><br><span class="line">systemctl restart mariadb <span class="comment">#重启MariaDB</span></span><br><span class="line">systemctl <span class="built_in">enable</span> mariadb <span class="comment">#设置开机启动</span></span><br></pre></td></tr></table></figure>
-<h3 id="正式安装mysql">正式安装MySQL</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#下载，在这里使用的是命令行下载，也建议在图形界面下载，然后上传至服务器</span></span><br><span class="line">wget http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm</span><br><span class="line"></span><br><span class="line"><span class="comment">#使用rpm安装</span></span><br><span class="line">rpm -ivh mysql-community-release-el7-5.noarch.rpm</span><br><span class="line"></span><br><span class="line"><span class="comment">#使用yum安装mysql-community-server</span></span><br><span class="line">yum install mysql-community-server</span><br><span class="line"></span><br><span class="line"><span class="comment">#启动服务</span></span><br><span class="line">service mysqld start</span><br><span class="line"></span><br><span class="line"><span class="comment">#进入mysql，第一次进入无密码</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#命令行变成如下</span></span><br><span class="line">mysql&gt;</span><br></pre></td></tr></table></figure>
-<h3 id="配置mysql">配置MySQL</h3>
-<p>编码配置 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入配置文件，若未安装vim，建议先使用命令yum install vim安装vim</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"></span><br><span class="line"><span class="comment">#最后加上编码配置</span></span><br><span class="line">[mysql]</span><br><span class="line">default-character-set =utf8</span><br><span class="line"></span><br><span class="line"><span class="comment">#此处字符编码必须和/usr/share/mysql/charsets/Index.xml中一致。</span></span><br><span class="line"><span class="comment">#不过一般情况下使用的都是utf8</span></span><br></pre></td></tr></table></figure></p>
-<p>设置密码 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#下面三种方法需要进入mysql</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#方法一</span></span><br><span class="line">mysql&gt;insert into user(host,user,password) values(<span class="string">&#x27;%&#x27;</span>,<span class="string">&#x27;user_name&#x27;</span>,password(<span class="string">&quot;password&quot;</span>);</span><br><span class="line"><span class="comment">#方法二</span></span><br><span class="line">mysql&gt;<span class="built_in">set</span> password <span class="keyword">for</span> user_name = password(<span class="string">&quot;password&quot;</span>);</span><br><span class="line"><span class="comment">#方法三</span></span><br><span class="line">mysql&gt;grant all on *.* to user_name@% identified by <span class="string">&quot;password&quot;</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">#下面这一种方法可直接在shell下设置密码</span></span><br><span class="line">mysqladmin -u root password <span class="string">&quot;password&quot;</span></span><br></pre></td></tr></table></figure> 远程连接</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入mysql</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#把在所有数据库的所有表的所有权限赋值给位于所有IP地址的root用户。</span></span><br><span class="line">mysql&gt; grant all privileges on *.* to root@<span class="string">&#x27;%&#x27;</span>identified by <span class="string">&quot;password&quot;</span>;</span><br></pre></td></tr></table></figure>
-<p>P.S.上文引号中的user_name表示数据库的用户名，password表示对应用户的密码。即这两项是由读者自行定义的。</p>
-<h3 id="常见问题及解决方案">常见问题及解决方案</h3>
-<h4 id="mysql创建用户失败">Mysql创建用户失败</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># mysql在my.ini的配置文件中设置了严格模式，所以我们需要进行修改</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 第一步，寻找配置文件</span></span><br><span class="line">whereis my.ini</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第二步，根据上一步结果进行vim编辑</span></span><br><span class="line">vim /../my.ini</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第三步，在vim里面搜索sql-mode,删除STRICT_TRANS_TABLES，保存退出即可</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 第四步，保险起见，使刚刚的配置文件立即生效</span></span><br><span class="line"><span class="built_in">source</span> /../my.ini</span><br></pre></td></tr></table></figure>
-<h4 id="支持中文">支持中文</h4>
-<ul>
-<li><p>Centos7</p>
-<p><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 编辑文件</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"><span class="comment"># 在对应[xx]下增加修改如下代码</span></span><br><span class="line">[client]</span><br><span class="line">port = 3306</span><br><span class="line">socket = /var/lib/mysql/mysql.sock</span><br><span class="line">default-character-set=utf8</span><br><span class="line"></span><br><span class="line">[mysqld]</span><br><span class="line">port = 3306</span><br><span class="line">socket = /var/lib/mysql/mysql.sock</span><br><span class="line">default-storage-engine=INNODB</span><br><span class="line">character-set-server=utf8</span><br><span class="line">collation-server=utf8_general_ci</span><br><span class="line"></span><br><span class="line">[mysql]</span><br><span class="line">no-auto-rehash</span><br><span class="line">default-character-set=utf8</span><br><span class="line"><span class="comment"># 保存退出，重启服务</span></span><br><span class="line"><span class="comment"># 重新登陆mysql检查是否成功，方法见下</span></span><br></pre></td></tr></table></figure></p></li>
-<li><p>Ubuntu16.04</p>
-<p><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">sudo</span> vim /etc/mysql/mysql.conf.d/mysqld.cnf</span><br><span class="line"><span class="comment"># 在对应[xx]下增加以下内容，如果不存在[xx]自行增加</span></span><br><span class="line">[mysqld]</span><br><span class="line">character_set_server=utf8</span><br><span class="line">[mysql]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[mysql.server]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[mysqld_safe]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[client]</span><br><span class="line">default-character-set=utf8</span><br><span class="line"><span class="comment"># 重启mysql</span></span><br><span class="line">service mysql restart</span><br><span class="line"><span class="comment"># 进入mysql查看配置参数</span></span><br><span class="line">mysql -uroot -p</span><br><span class="line"><span class="comment"># 查看database 的value变为utf8即可</span></span><br><span class="line">&gt;show variables like <span class="string">&#x27;%character%&#x27;</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment"># p.s.如果之前创建了表，表编码不会改变</span></span><br></pre></td></tr></table></figure></p></li>
-</ul>
-<h4
-id="mysql报错1366---incorrect-integer-value-xxxxxxx-at-row-1">MySQL报错“1366
-- Incorrect integer value: '' XXXXXXX' at row 1 ”</h4>
-<p>修改方法:（两种,建议第二种）</p>
-<ol type="1">
-<li><p>命令行。set names gbk；(此为设置通信编码)</p></li>
-<li><p><code>my.cnf</code> 中查找sql-mode</p></li>
-</ol>
-<p>将 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">sql-mode=<span class="string">&quot;STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION&quot;</span>，</span><br></pre></td></tr></table></figure> 修改为 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">sql-mode=<span class="string">&quot;NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION&quot;</span>，</span><br></pre></td></tr></table></figure> 重启mysql后即可</p>
-<h4 id="关于my.cnf和my.ini的说明">关于my.cnf和my.ini的说明</h4>
-<p>my.cnf常见于Linux系统，my.ini常见与windows系统，二者都是属于mysql的配置文件。一般好像在一个系统下就是只出现一种配置文件，具体区别没有深入了解，修改配置文件，根据自己的系统进行查找修改配置文件即可</p>
-<h4 id="mysql交互环境自动补全">Mysql交互环境自动补全</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 修改配置文件</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"><span class="comment"># 在[mysql]部分添加auto-rehash</span></span><br><span class="line"><span class="comment"># 保存退出，重启</span></span><br><span class="line">service mysqld restart</span><br></pre></td></tr></table></figure>
-<h2 id="配置hive">配置Hive</h2>
-<p>我们之前在hadoop目录下安装了Hive，位置为<code>/usr/local/hadoop/apache-hive-1.2.1-bin</code></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入hive配置目录下</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop/apache-hive-1.2.1-bin/conf</span><br><span class="line"></span><br><span class="line"><span class="comment">#修改hive-default.xml.template</span></span><br><span class="line"><span class="comment">#首先复制</span></span><br><span class="line"><span class="built_in">cp</span> hive-default.xml.template hive-default.xml</span><br><span class="line"><span class="comment">#修改文件</span></span><br><span class="line">vim hive-default.xml</span><br><span class="line"></span><br><span class="line"><span class="comment"># 1. 第一步将&lt;configuration&gt;&lt;/configuration&gt;中内容删除大概是18-3908行</span></span><br><span class="line"><span class="comment"># vim删除命令--&gt;:18,3908d</span></span><br><span class="line"><span class="comment"># 2.将下面&lt;configuration&gt;&lt;/configuration&gt;中内容复制进去,分别将3306喝9083前的user_name改为当前的用户名</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionURL&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;jdbc:mysql://user_name:3306/hive?createDatabaseIfNotExist=<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;JDBC connect string <span class="keyword">for</span> a JDBC metastore&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionDriverName&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;com.mysql.jdbc.Driver&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;Driver class name <span class="keyword">for</span> a JDBC metastore&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line"></span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionUserName&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hive&lt;value&gt;</span><br><span class="line">        &lt;description&gt;username to use against metastore database&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionPassword&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hive&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;password to use against metastore database&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line"></span><br><span class="line">    &lt;property&gt;</span><br><span class="line">    &lt;name&gt;hive.metastore.uris&lt;/name&gt;</span><br><span class="line">    &lt;value&gt;thrift://user_name:9083&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br></pre></td></tr></table></figure>
-<h2 id="下载jdbc">下载JDBC</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#命令行下载,也可以使用图形界面上传文件</span></span><br><span class="line"> wget http://cdn.mysql.com/Downloads/Connector-J/mysql-connector-java-5.1.36.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#将文件复制进Hive的lib目录下，原因，Hive的自带的那个版本低，可能失效</span></span><br><span class="line"> <span class="built_in">cp</span> mysql-connector-java-5.1.33-bin.jar /usr/local/hadoop/apache-hive-1.2.1-bin/lib/</span><br></pre></td></tr></table></figure>
-<p>启动Hive <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">hive --service metastore &amp;</span><br><span class="line">jps</span><br><span class="line"><span class="comment">#结果会多出一个进程</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#进入hive目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop/apache-hive-1.2.1-bin/bin</span><br><span class="line"><span class="comment">#启动hive，可能有点慢</span></span><br><span class="line">hive</span><br><span class="line"></span><br><span class="line"><span class="comment">#若出现hive的命令行即代表成功，如下</span></span><br><span class="line">hive&gt;</span><br></pre></td></tr></table></figure></p>
-<h2 id="常见问题及解决方案-1">常见问题及解决方案</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">Logging initialized using configuration <span class="keyword">in</span> jar:file:/home/hadoop/apache-hive-1.2.1-bin/lib/hive-common-1.2.1.jar!/hive-log4j.properties</span><br><span class="line">Exception <span class="keyword">in</span> thread <span class="string">&quot;main&quot;</span> java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient</span><br><span class="line">        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:522)</span><br></pre></td></tr></table></figure>
-<p>解决方法，直接关闭防火墙</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#这里的系统为Centos7，所以使用此命令</span></span><br><span class="line">systemctl stop firewalld</span><br></pre></td></tr></table></figure>
-<p>hive metastore 启动出错解决 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看与hive相关进程是否启动</span></span><br><span class="line">ps -ef | grep hive</span><br><span class="line"><span class="comment"># kill相关进程，为进程号</span></span><br><span class="line"><span class="built_in">kill</span> num</span><br><span class="line"><span class="comment"># 重新启动</span></span><br><span class="line">./hive</span><br></pre></td></tr></table></figure></p>
-<p>更多问题见此文章 <a
-href="http://blog.csdn.net/freedomboy319/article/details/44828337">Hive常见问题汇总</a></p>
-<h2 id="hive的学习笔记">Hive的学习笔记</h2>
-<h3 id="新建表">新建表</h3>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 新建一张表，名为“test”，里面有“name”、“id”两类，分别是“string”、“int”的数据类型，以“|”隔开一列，表是作为textfile的。</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">CREATE</span> <span class="keyword">TABLE</span> test (name String, id <span class="type">int</span>) <span class="type">ROW</span> FORMAT DELIMITED FIELDS TERMINATED <span class="keyword">BY</span> <span class="string">&#x27;|&#x27;</span> STORED <span class="keyword">AS</span> TEXTFILE;</span><br></pre></td></tr></table></figure>
-<h3 id="加载表">加载表</h3>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 从本地的/home/user/test.txt文件，将数据加载进test这个表</span></span><br><span class="line">hive<span class="operator">&gt;</span> LOAD DATA <span class="keyword">LOCAL</span> INPATH <span class="string">&#x27;/home/user/test.txt&#x27;</span> OVERWRITE <span class="keyword">INTO</span> <span class="keyword">TABLE</span> test;</span><br></pre></td></tr></table></figure>
-<h3 id="关联表">关联表</h3>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 将两张表通过一个或多个字段关联在一起</span></span><br><span class="line"><span class="comment">-- 将test_a和test_b表的id字段关联在一起，再加上“name”字段，组成一个新表</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> test_a.id, test_b.name <span class="keyword">FROM</span> test_a, test_b <span class="keyword">JOIN</span> test_b <span class="keyword">ON</span> (test_a.id <span class="operator">=</span> test_b.id);</span><br></pre></td></tr></table></figure>
-<h3 id="保存表">保存表</h3>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 由于hive下执行任务之后，并不会保存数据，所以我们使用INSERT命令来保存命令</span></span><br><span class="line"><span class="comment">-- 插入数据到test表中</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">INSERT</span> OVERWRITE <span class="keyword">TABLE</span> test</span><br><span class="line">     <span class="operator">&gt;</span> ...</span><br><span class="line"></span><br><span class="line"><span class="comment">-- 保存表到hdfs上,保存输出结果到hdfs下的/out/目录下</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">INSERT</span> OVERWRITE <span class="keyword">LOCAL</span> DIRECTORY &quot;/out/&quot;</span><br><span class="line">    <span class="operator">&gt;</span> ...</span><br></pre></td></tr></table></figure>
-<h3 id="排序问题">排序问题</h3>
-<ol type="1">
-<li>order by</li>
-</ol>
-<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test <span class="keyword">ORDER</span> <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure>
-<ol start="2" type="1">
-<li><p>sort by <figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test SORT <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure></p></li>
-<li><p>distribute by <figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test <span class="keyword">ORDER</span> <span class="keyword">BY</span> name DISTRIBUTE <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure></p></li>
-<li><p>DISTRIBUTE BY with SORT BY DISTRIBUTE BY 和 GROUP BY
-有点类似，DISTRIBUTE BY控制reduce如何处理数据，而SORT
-BY控制reduce中的数据如何排序。 注意：hive要求DISTRIBUTE BY语句出现在SORT
-BY语句之前。</p></li>
-<li><p>Cluster By</p></li>
-</ol>
-<p>cluster by 除了具有 distribute by 的功能外还兼具 sort by 的功能。
-默认升序排序，但DISTRIBUTE BY的字段和SORT
-BY的字段必须相同，且不能指定排序规则。</p>
-<p>总结：</p>
-<ul>
-<li><code>ORDER BY</code>是全局排序，但在数据量大的情况下，花费时间会很长</li>
-<li><code>SORT BY</code>是将<code>reduce</code>的单个输出进行排序，不能保证全局有序</li>
-<li><code>DISTRIBUTE BY</code>
-可以按指定字段将数据划分到不同的<code>reduce</code>中</li>
-<li>当<code>DISTRIBUTE BY</code> 的字段和
-<code>SORT BY</code>的字段相同时，可以用<code>CLUSTER BY</code>来代替<code>DISTRIBUTE BY with SORT BY</code>。</li>
-</ul>
-<h3 id="hive常用函数">hive常用函数</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 以下无特殊说明，返回值皆为string</span></span><br><span class="line"><span class="comment"># 字符串连接</span></span><br><span class="line">hive&gt;concat(strA, strB, ...)</span><br><span class="line"><span class="comment"># 带分隔符的字符串连接，即连接之后字符串存在分隔符，strSEP表示分隔符</span></span><br><span class="line">hive&gt;concat_wd(strSEP, strA, strB, ...)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串截取</span></span><br><span class="line"><span class="comment"># 从整型数num0开始截取，截取到结尾为止</span></span><br><span class="line">hive&gt;substr(strA, num0)</span><br><span class="line"><span class="comment"># 从整型数num0开始截取，截取到num1为止</span></span><br><span class="line">hive&gt;substr(strA, num0, num1)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串转大写</span></span><br><span class="line">hive&gt;upper(str)</span><br><span class="line"><span class="comment"># 字符串转小写</span></span><br><span class="line">hive&gt;lower(str)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 除去字符串中空格</span></span><br><span class="line">hive&gt;trim(str)</span><br><span class="line"><span class="comment"># 除去字符串左边空格</span></span><br><span class="line">hive&gt;ltrim(str)</span><br><span class="line"><span class="comment"># 除去字符串右边空格</span></span><br><span class="line">hive&gt;rtrim(str)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则替换字符</span></span><br><span class="line"><span class="comment"># 将strA中符合strRE的部分替换为strB</span></span><br><span class="line">hive&gt;regexp_replace(strA, strRE, strB)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重复字符串</span></span><br><span class="line"><span class="comment"># str为字符串，num为重复次数</span></span><br><span class="line">hive&gt;repeat(str, num)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 分割字符串</span></span><br><span class="line"><span class="comment"># 以strpat为分割符，返回字符串数组</span></span><br><span class="line">hive&gt;spilt(str, strpat)</span><br></pre></td></tr></table></figure>
-<h3 id="hive导出数据">hive导出数据</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导出至本地，content表示本地目录</span></span><br><span class="line">hive&gt;insert overwrite <span class="built_in">local</span> directory <span class="string">&#x27;content&#x27;</span></span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 导出至hdfs，content表示hdfs目录</span></span><br><span class="line">hive&gt;insert overwrite directory <span class="string">&#x27;content&#x27;</span></span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 导出至hive另一张表othertable，前提hive中存在这张表</span></span><br><span class="line">hive&gt;insert overwrite othertable</span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br></pre></td></tr></table></figure>
-<h3 id="转换类型">转换类型</h3>
-<p>cast(xxx as xx) 将xxx类型转换为xx类型</p>
-<p>下表为是否可转换类型的说明 <img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/hadoop/20170612195815151.png" /></p>
-<p>注：由于表格比较大，这里对一些比较长的字符串进行缩写，ts是timestamp的缩写,bl是boolean的缩写,sl是smallint的缩写,dm是decimal的缩写,vc是varchar的缩写,ba是binary的缩写。</p>
+<h2 id="滑动窗口">滑动窗口</h2>
+<p><a
+href="https://leetcode.cn/problems/longest-substring-without-repeating-characters/">3.
+无重复字符的最长子串</a></p>
+<p>从左向右滑，需要一个数据结构（A）记录已经遍历的字符，还有一个变量（B）记录最大长度</p>
+<p>注：每次不需要从头开始，而是可以从重复那个字符的后一个开始，所以A需要记录一下索引</p>
+<h2 id="kmp">KMP</h2>
+<h2 id="动态规划">动态规划</h2>
+<p><a
+href="https://leetcode.cn/problems/longest-palindromic-substring/">5.
+最长回文子串</a></p>
+<p>两个指针从两头开始，所以需要二维DP，元素是bool判断</p>
+<p><a href="https://leetcode.cn/problems/interleaving-string/">97.
+交错字符串</a></p>
+<p>两个字符串组成新的字符串，二维DP，元素是bool判断</p>
+<p><a href="https://leetcode.cn/problems/word-break/">139.
+单词拆分</a></p>
+<p>单词数组，一个一个单词拼接，组成最后单词，一维DP，元素是bool判断</p>
+<p><a
+href="https://leetcode.cn/problems/longest-palindromic-subsequence/">516.
+最长回文子序列</a></p>
+<p>回文，需要两个指针，二维DP，从i到j组成回文的长度，元素是长度，最小长度为1，相等+2</p>
+<p><a href="https://leetcode.cn/problems/ones-and-zeroes/">474.
+一和零</a></p>
+<p>字符串数组，组成数量限制（两个限制），所以限制需要二个维度，数组作为最外层那个维度，元素是个数，最小为0</p>
+<p><a
+href="https://leetcode.cn/problems/delete-operation-for-two-strings/">583.
+两个字符串的删除操作</a></p>
+<p>两个字符串，两个维度，删除次数，初始情况，删除个数为字符串长度。</p>
+<p><a href="https://leetcode.cn/problems/palindromic-substrings/">647.
+回文子串</a></p>
+<p>回文，二维DP，元素是bool判断，统计True个数</p>
+<p><a
+href="https://leetcode.cn/problems/longest-common-subsequence/">1143.
+最长公共子序列</a></p>
+<p>两个字符串，二维DP，长度</p>
+<p><a href="https://leetcode.cn/problems/jump-game-vii/">1871. 跳跃游戏
+VII</a></p>
+<p>能否到达终点，一维DP，反向思考，能否到达该点，所以元素为bool。+前缀和优化时间</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 回文串</span></span><br><span class="line">dp = [[<span class="literal">False</span> <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)] <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(n)]</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(n):</span><br><span class="line">    dp[i][i] = <span class="literal">True</span></span><br><span class="line">    <span class="comment"># 从i到j是否组成回文串</span></span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(n-<span class="number">1</span>)[::-<span class="number">1</span>]:</span><br><span class="line">        <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(i+<span class="number">1</span>, n):</span><br><span class="line">            <span class="keyword">if</span> s[i] == s[j] <span class="keyword">and</span> (j - i &lt;= <span class="number">2</span> <span class="keyword">or</span> dp[i+<span class="number">1</span>][j-<span class="number">1</span>]):</span><br><span class="line">                dp[i][j] = <span class="literal">True</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h2 id="双指针">双指针</h2>
+<p><a
+href="https://leetcode.cn/problems/longest-repeating-character-replacement/">424.
+替换后的最长重复字符</a></p>
+<p>枚举右端点，如果没超过限制就继续移动右指针，否则移动左指针</p>
+<h2 id="栈">栈</h2>
+<p>PASS</p>
+<h2 id="字典树">字典树</h2>
+<p>PASS</p>
+<h2 id="模拟深搜">模拟/深搜</h2>
+<p>PASS</p>
 ]]></content>
       <categories>
-        <category>Environment</category>
+        <category>Algorithm</category>
       </categories>
       <tags>
-        <tag>Hadoop</tag>
-        <tag>大数据</tag>
-        <tag>Hive</tag>
+        <tag>字符串</tag>
       </tags>
   </entry>
   <entry>
-    <title>crontab命令</title>
-    <url>/hexo_blog/2017/08/24/%E8%BF%90%E7%BB%B4/crontab%E5%91%BD%E4%BB%A4/</url>
-    <content><![CDATA[<p>学会使用crontab定时任务，提高工作效率。详解了crontab的基本格式和例子，以及相关目录介绍和日志文件的设置。</p>
+    <title>Mongodb：副本集配置</title>
+    <url>/hexo_blog/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/</url>
+    <content><![CDATA[<p>Mongodb4.2.x数据库配置的第二部分，副本集架构的配置</p>
 <span id="more"></span>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看当前用户的定时任务，也可以使用crontab -uroot -l查看指定用户的定时任务</span></span><br><span class="line">crontab -l</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编辑crontab定时任务，貌似是使用vim的编辑命令</span></span><br><span class="line">crontab -e</span><br><span class="line"></span><br><span class="line"><span class="comment"># 其实基本上知道这两个就可以处理定时任务了</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除当前用户的所有定时任务，不知道为什么会有这种设定（手贱按下过一次，恢复起来真的惨）</span></span><br><span class="line">crontab -r</span><br><span class="line"><span class="comment"># 如果按到了，就去日志文件里面找记录，尽可能恢复命令吧</span></span><br></pre></td></tr></table></figure>
-<p>下面讲讲编辑任务的格式</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 基本格式</span></span><br><span class="line"></span><br><span class="line">  *  *  *  *  * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 分 时 日 月 周 需要执行的命令</span></span><br></pre></td></tr></table></figure>
-<p>可视化测试在线网站，https://crontab.guru/</p>
-<p>几个栗子</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 每分钟执行一次command命令，空格表示间隔</span></span><br><span class="line">* * * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每小时执行一次，‘/’表示频率</span></span><br><span class="line">0 */1 * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每小时的10分和40分各执行一次，‘,’表示并列</span></span><br><span class="line">10,40 * * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每个星期一的10-14点的10，40执行，‘-’表述区间</span></span><br><span class="line">10,40 10-14 * * 1 <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每天的8，13，20点执行，注意此时表示分的那个*要替换为0</span></span><br><span class="line">0 8,13,20 * * * <span class="built_in">command</span></span><br></pre></td></tr></table></figure>
-<p>日志文件一般存放在
-<code>/var/log/cron*</code>中，不过我的Ubuntu好像没有开启cron日志，可以在
-<code>/etc/rsyslog.d/50-default.conf</code>中进行设置（Centos在
-<code>/etc/rsyslog.conf</code>中设置）</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">&gt; <span class="built_in">sudo</span> vim /etc/rsyslog.d/50-default.conf</span><br><span class="line"><span class="comment"># 找到cron*, 去掉前面的#注释即可</span></span><br><span class="line"><span class="comment"># 重启rsyslog</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> service rsyslog  restart</span><br></pre></td></tr></table></figure>
-<p>crontab相关目录介绍</p>
-<p><code>/etc/anacrontab</code>:
-这个文件存着系统级的任务。它主要用来运行每日的(daily)，每周的(weekly),每月的(monthly)的任务。一般不在此文件安装自己的任务</p>
-<p><code>/etc/cron.d/</code>: 此目录下存放的是系统级任务的任务文件。</p>
-<p><code>/var/spool/cron/</code>:
-此目录下存放各个用户的任务文件。各个用户的任务存放在以自已用户名为文件名的任务文件中。此文件中的指令行没有用户域。</p>
-<p><code>/etc/crontab</code>:
-crontab的主要配置文件，执行了每小时、每日、每周、每月的脚本文件</p>
-<p><code>/etc/cron.hourly</code>、<code>/etc/cron.daily</code>、<code>/etc/cron.weekly</code>、<code>/etc/cron.monthly</code>：这四个目录下分别对应着每隔一段时间需要执行的脚本任务，脚本使用shell命令编写</p>
-<p><a href="https://segmentfault.com/a/1190000002628040"></a></p>
-<p><a
-href="http://www.cnblogs.com/peida/archive/2013/01/08/2850483.html">每天一个linux命令（50）：crontab命令</a></p>
-<p><a href="https://segmentfault.com/a/1190000002628040">Linux
-下执行定时任务 crontab 命令详解</a></p>
+<h2 id="准备">准备</h2>
+<p>接
+<a href="/hexo_blog/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/" title="Mongodb从安装到基本操作">Mongodb从安装到基本操作</a>，基本同原文，由于是副本集配置，所以在需要配置的服务器上进行相同的配置</p>
+<p>选择：</p>
+<ol type="1">
+<li>单机上，配置不同port的mongo服务</li>
+<li>不同机器上，配置mongo服务</li>
+<li>要求同网段，所以配置的ip中不能写127.0.0.1</li>
+<li>可以 <code>vim /etc/hosts</code>，方便管理</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载压缩包</span></span><br><span class="line">wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-4.2.15.tgz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 解压缩</span></span><br><span class="line">tar -zxvf mongodb-linux-x86_64*.tgz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重命名</span></span><br><span class="line"><span class="built_in">mv</span> mongodb-linux-x86_64* mongodb</span><br><span class="line"><span class="comment"># 移动到你想放置的目录下，这里我放在/opt目录下</span></span><br><span class="line"><span class="built_in">mv</span> mongodb /opt/</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行mongodb主目录</span></span><br><span class="line"><span class="built_in">cd</span> /opt/</span><br><span class="line"><span class="comment"># 建立存放数据文件和日志文件的目录</span></span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/logs</span><br><span class="line"><span class="built_in">mkdir</span> -p data/test/db</span><br><span class="line"><span class="comment"># 创建配置文件，并写入如下配置</span></span><br><span class="line">vim bin/mongodb.conf</span><br></pre></td></tr></table></figure>
+<p>更新部分，配置文件用yaml格式</p>
+<figure class="highlight yaml"><table><tr><td class="code"><pre><span class="line"><span class="attr">systemLog:</span></span><br><span class="line">  <span class="attr">destination:</span> <span class="string">file</span></span><br><span class="line">  <span class="attr">path:</span> <span class="string">&quot;/opt/mongodb/data/test/logs/mongodb.log&quot;</span></span><br><span class="line">  <span class="attr">logAppend:</span> <span class="literal">true</span></span><br><span class="line"><span class="attr">storage:</span></span><br><span class="line">  <span class="attr">dbPath:</span> <span class="string">&quot;/opt/mongodb/data/test/db&quot;</span></span><br><span class="line">  <span class="attr">journal:</span></span><br><span class="line">    <span class="attr">enabled:</span> <span class="literal">true</span> </span><br><span class="line"><span class="attr">net:</span></span><br><span class="line">  <span class="comment"># 允许所有IP访问，也可以用逗号隔开，控制IP访问</span></span><br><span class="line">  <span class="attr">bindIp:</span> <span class="number">0.0</span><span class="number">.0</span><span class="number">.0</span> </span><br><span class="line">  <span class="comment"># 设置端口号（默认的端口号是27017，可以根据个人需求进行更改）</span></span><br><span class="line">  <span class="attr">port:</span> <span class="number">27017</span></span><br><span class="line"><span class="attr">processManagement:</span></span><br><span class="line">  <span class="comment"># 设置为以守护进程的方式运行，即在后台运行</span></span><br><span class="line">  <span class="attr">fork:</span> <span class="literal">true</span></span><br><span class="line"><span class="attr">replication:</span></span><br><span class="line">  <span class="comment"># 副本集名字</span></span><br><span class="line">  <span class="attr">replSetName:</span> <span class="string">&quot;rs0&quot;</span> </span><br></pre></td></tr></table></figure>
+<h3 id="启动">启动</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在所有机器上启动mongo，此时无账号密码</span></span><br><span class="line"><span class="comment"># 以配置文件的方式启动</span></span><br><span class="line">bin/mongod --config mongodb.conf</span><br></pre></td></tr></table></figure>
+<h3 id="配置副本集">配置副本集</h3>
+<p>在任意一台服务器A上，进行配置副本集</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 将副本集的初始化配置写入js</span></span><br><span class="line">vim init_replica.js</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 其中rs0一定要跟上面的replSetName一样</span></span><br><span class="line">config = &#123; _id:<span class="string">&quot;rs0&quot;</span>, members:[&#123;_id:0,host:<span class="string">&quot;ip0:port0&quot;</span>&#125;,&#123;_id:1,host:<span class="string">&quot;ip1:port1&quot;</span>&#125;]&#125;</span><br><span class="line">rs.initiate(config) </span><br><span class="line">rs.status()</span><br></pre></td></tr></table></figure>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/mongod --port 27017 &lt; init_replica.js</span><br></pre></td></tr></table></figure>
+<h3 id="验证">验证</h3>
+<p>在A服务器上进行操作，添加账号密码</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行mongodb的交互环境</span></span><br><span class="line">bin/mongod --port 27017</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行admin数据库，创建管理员用户root，密码为password，权限是超级用户（最高）</span></span><br><span class="line">&gt;use admin</span><br><span class="line">&gt;db.createUser(&#123; user: <span class="string">&quot;admin&quot;</span>, <span class="built_in">pwd</span>: <span class="string">&quot;password&quot;</span>, roles: [&#123; role: <span class="string">&quot;userAdminAnyDatabase&quot;</span>, db: <span class="string">&quot;admin&quot;</span> &#125;] &#125;)</span><br><span class="line">&gt;db.createUser(&#123;user:<span class="string">&quot;root&quot;</span>,<span class="built_in">pwd</span>:<span class="string">&quot;password&quot;</span>,roles:[<span class="string">&quot;root&quot;</span>]&#125;)</span><br><span class="line"><span class="comment"># 验证是否创建成功，返回1表示成功</span></span><br><span class="line">&gt;db.auth(&#123;<span class="string">&quot;root&quot;</span>, <span class="string">&quot;password&quot;</span>&#125;)</span><br><span class="line"><span class="comment"># 退出交互环境</span></span><br><span class="line">&gt;<span class="built_in">exit</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 找到mongo的服务进程pid</span></span><br><span class="line">ps ux | grep mongod</span><br><span class="line"><span class="comment"># 杀死进程，填写对应的pid</span></span><br><span class="line"><span class="built_in">kill</span> PID</span><br><span class="line"></span><br><span class="line"><span class="comment"># 以密码权限验证启动服务</span></span><br><span class="line">bin/mongod --config mongodb.conf --auth</span><br></pre></td></tr></table></figure>
+<h3 id="openssl证书文件">OpenSSL证书文件</h3>
+<h3 id="读写分离">读写分离</h3>
+<h3 id="主节点选举配置">主节点选举配置</h3>
 ]]></content>
       <categories>
-        <category>Software</category>
+        <category>Database</category>
       </categories>
       <tags>
-        <tag>Linux</tag>
-        <tag>crontab</tag>
+        <tag>mongodb</tag>
+        <tag>数据库</tag>
       </tags>
   </entry>
   <entry>
-    <title>风格迁移转换的两种实现</title>
-    <url>/hexo_blog/2017/08/14/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E9%A3%8E%E6%A0%BC%E8%BF%81%E7%A7%BB%E8%BD%AC%E6%8D%A2%E7%9A%84%E4%B8%A4%E7%A7%8D%E5%AE%9E%E7%8E%B0/</url>
-    <content><![CDATA[<p>本文介绍了如何在Ubuntu16.04环境下搭建caffe和tensorflow，并利用caffe和tensorflow实现了风格转换。同时，介绍了如何训练新的风格模型。</p>
+    <title>Python魔术方法</title>
+    <url>/hexo_blog/2018/03/14/Python/Python%E9%AD%94%E6%9C%AF%E6%96%B9%E6%B3%95/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Python的魔术方法</p>
 <span id="more"></span>
-<p><strong>style-transfer</strong></p>
-<p>这个主要是利用caffe+model+code直接实现任意风格的转换。</p>
-<p>最后实现需借助</p>
-<ul>
-<li>一张风格图片</li>
-<li>待转换风格的目标图片</li>
-<li>训练模型</li>
-</ul>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">python style.py -s &lt;style_image&gt; -c &lt;content_image&gt; -m &lt;model_name&gt; -g 0</span><br></pre></td></tr></table></figure>
-<p>优势：</p>
-<ul>
-<li>可以实现任意风格的转换</li>
-<li>可拓展性强 缺点：</li>
-<li>耗时长（真的很长，如果用CPU的话）</li>
-</ul>
-<p><strong>fast-neural-style-tensorflow</strong></p>
-<p>风格快速迁移转换</p>
-<p>顾名思义，这个比前者能够更快的进行风格转换，速度因机器而异，不过效率比前者确确实实提高了很多倍</p>
-<p>最后实现需借助</p>
-<ul>
-<li>一个训练好的风格模型</li>
-<li>待转换的风格图片</li>
-</ul>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding</span></span><br><span class="line"> python eval.py --model_file &lt;path of ckpt-done&gt; --image_file &lt;path of image&gt;</span><br></pre></td></tr></table></figure>
-<p>优势：</p>
+<p>参考文章:</p>
+<p><a
+href="http://pycoders-weekly-chinese.readthedocs.io/en/latest/issue6/a-guide-to-pythons-magic-methods.html">Python
+魔术方法指南</a></p>
+<p>魔术方法，顾名思义是一种可以给对象(类)增加魔法的特殊方法，它们的表示方法一般是用双下划线包围(如<code>__init__</code>)</p>
+<h2 id="对象实例化">对象实例化</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> os.path <span class="keyword">import</span> join</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">FileObject</span>:</span><br><span class="line">    <span class="string">&#x27;&#x27;&#x27;给文件对象进行包装从而确认在删除时文件流关闭&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, filepath=<span class="string">&#x27;~&#x27;</span>, filename=<span class="string">&#x27;sample.txt&#x27;</span></span>):</span><br><span class="line">        <span class="comment">#读写模式打开一个文件</span></span><br><span class="line">        <span class="variable language_">self</span>.file = <span class="built_in">open</span>(join(filepath, filename), <span class="string">&#x27;r+&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__del__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="variable language_">self</span>.file.close()</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>.file</span><br></pre></td></tr></table></figure>
 <ul>
-<li>速度相对来说很快</li>
-<li>环境搭建相对来说更容易 缺点：</li>
-<li>可拓展性更弱</li>
-<li>只能转换固定几种风格</li>
-<li>训练新风格模型时间长（比前者生成时间更长）</li>
+<li><p><code>__new__(cls, [...])</code>:
+在一个对象实例化调用的第一个方法，第一个参数是类，剩下的参数传递给<code>__init__</code></p></li>
+<li><p><code>__init__(self, [...])</code>:
+类的初始化方法。当构造函数被调用的时候的任何参数够会传给它</p></li>
+<li><p><code>__del__(self)</code>:
+<code>__new__</code>和<code>__init__</code>为对象的构造器，<code>__del__</code>为对象的析构器。当一个对象进行过垃圾回收时候的行为就是这个方法。当一个对象在删除的时候需要更多的清洁工作，就需要使用这个方法</p></li>
 </ul>
-<h2 id="style-transfer的实现">style-transfer的实现</h2>
-<p><strong>环境介绍</strong></p>
-<p>Ubuntu16.04 + CPU + python2.7 + caffe</p>
-<h3 id="搭建caffe环境">搭建caffe环境</h3>
-<h4 id="介绍">介绍</h4>
-<p>Caffe是一个清晰而高效的深度学习框架。Caffe是纯粹的C++/CUDA架构，支持命令行、Python和MATLAB接口；可以在CPU和GPU直接无缝切换。</p>
-<p>优势：</p>
+<h2 id="比较">比较</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Word</span>(<span class="title class_ inherited__">str</span>):</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;存储单词的类，定义比较单词的几种方法&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__new__</span>(<span class="params">cls, word</span>):</span><br><span class="line">        <span class="comment"># 注意我们必须要用到__new__方法，因为str是不可变类型</span></span><br><span class="line">        <span class="comment"># 所以我们必须在创建的时候将它初始化</span></span><br><span class="line">        <span class="keyword">if</span> <span class="string">&#x27; &#x27;</span> <span class="keyword">in</span> word:</span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&quot;Value contains spaces. Truncating to first space.&quot;</span>)</span><br><span class="line">            word = word[:word.index(<span class="string">&#x27; &#x27;</span>)] <span class="comment">#单词是第一个空格之前的所有字符</span></span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">str</span>.__new__(cls, word)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__gt__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &gt; <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__lt__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &lt; <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__ge__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &gt;= <span class="built_in">len</span>(other)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__le__</span>(<span class="params">self, other</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>) &lt;= <span class="built_in">len</span>(other)</span><br></pre></td></tr></table></figure>
 <ul>
-<li>上手快：模型与相应优化都是以文本形式而非代码形式给出。
-Caffe给出了模型的定义、最优化设置以及预训练的权重，方便立即上手。</li>
-<li>速度快：能够运行最棒的模型与海量的数据。
-Caffe与cuDNN结合使用，测试AlexNet模型，在K40上处理每张图片只需要1.17ms.</li>
-<li>模块化：方便扩展到新的任务和设置上。
-可以使用Caffe提供的各层类型来定义自己的模型。</li>
-<li>开放性：公开的代码和参考模型用于再现。</li>
-<li>社区好：可以通过BSD-2参与开发与讨论。</li>
+<li><p><code>__cmp__(self, other)</code>:
+用于比较的魔术方法，实现了所有的比较符号。如果<code>self</code> &lt;
+<code>other</code>返回一个负数；如果<code>self</code> ==
+<code>other</code>返回０；如果<code>self</code> &gt;
+<code>other</code>返回正数。一般情况下是分别定义每一个比较符号。如果想实现所有的比较符号，就需要调用这个方法。</p></li>
+<li><p><code>__eq__(self, other)</code>: 等号</p></li>
+<li><p><code>__ne__(self, other)</code>:　不等号</p></li>
+<li><p><code>__lt__(self, other)</code>:　小于</p></li>
+<li><p><code>__gt__(self, other)</code>:　大于</p></li>
+<li><p><code>__le__(self, other)</code>:　小于等于</p></li>
+<li><p><code>__ge__(self, other)</code>:　大于等于</p></li>
 </ul>
-<h4 id="安装依赖">安装依赖</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># caffe 依赖</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler</span><br><span class="line"><span class="built_in">sudo</span> apt-get install --no-install-recommends libboost-all-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libatlas-base-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libhdf5-serial-dev</span><br><span class="line"><span class="comment"># Python 依赖</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libgflags-dev libgoogle-glog-dev liblmdb-dev</span><br></pre></td></tr></table></figure>
-<h4
-id="安装caffe可利用virtualenv新建一个纯净环境">安装caffe（可利用virtualenv新建一个纯净环境）</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 我们需要将caffe的源码下载下来，所以需要git</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install git</span><br><span class="line"><span class="comment"># 下载代码</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/BVLC/caffe.git</span><br><span class="line"><span class="comment"># 下载完成后，进入caffe文件夹</span></span><br><span class="line"><span class="built_in">cd</span> caffe</span><br><span class="line"><span class="comment"># 安装caffe的python依赖</span></span><br><span class="line"><span class="keyword">for</span> req <span class="keyword">in</span> $(<span class="built_in">cat</span> requirements.txt); <span class="keyword">do</span> pip install <span class="variable">$req</span>; <span class="keyword">done</span></span><br></pre></td></tr></table></figure>
-<h4 id="配置caffe">配置caffe</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先确保在caffe文件夹中，然后修改配置文件</span></span><br><span class="line"><span class="comment"># 进行备份</span></span><br><span class="line"><span class="built_in">cp</span> Makefile.config.example Makefile.config</span><br><span class="line"><span class="comment"># 进行编辑</span></span><br><span class="line">vim Makefile.config</span><br></pre></td></tr></table></figure>
+<h2 id="一元操作符和函数">一元操作符和函数</h2>
 <ul>
-<li>因为CPU MODE, 所以在CPU_ONLY := 1前面的#要去掉</li>
-<li>两个路径要改成这样:(添加后面的两个hdf5的路径,
-否则编译时报hdf5错误)</li>
+<li><code>__pos__(self)</code>：实现正号的特性</li>
+<li><code>__neg__(self)</code>：实现负号的特性</li>
+<li><code>__abs__(self)</code>：abs, 绝对值</li>
+<li><code>__invert__(self)</code>：实现~的特性</li>
 </ul>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Whatever else you find you need goes here.</span></span><br><span class="line">INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial</span><br><span class="line">LIBRARY_DIRS := $(PYTHON_LIB) /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu/hdf5/serial</span><br></pre></td></tr></table></figure>
-<p>修改完成之后，保存退出即可</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行编译</span></span><br><span class="line">make pycaffe</span><br><span class="line">make all</span><br><span class="line">make <span class="built_in">test</span></span><br><span class="line">make runtest</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译结束后，需要设置环境变量</span></span><br><span class="line"><span class="comment"># 首先确保已经进入到caffe文件夹中，之后查看当前路径</span></span><br><span class="line"><span class="built_in">pwd</span></span><br><span class="line"><span class="comment"># 假设输出结果是xxx，那么将xxx复制下来，黏贴至下面xxx的位置</span></span><br><span class="line"><span class="built_in">export</span> PYTHONPATH=xxx/python:<span class="variable">$PYTHONPATH</span></span><br></pre></td></tr></table></figure>
-<h4 id="测试">测试</h4>
-<p>一般来说，如果运气好的话，到这里基本就算安装完成，但我们还是要测试一下</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进入python环境</span></span><br><span class="line">python</span><br><span class="line"><span class="comment"># 导入caffe模块</span></span><br><span class="line">&gt;caffe</span><br><span class="line"><span class="comment"># 如果此处没有报任何错误，恭喜你安装成功</span></span><br></pre></td></tr></table></figure>
-<h4 id="部分报错解决方案">部分报错解决方案</h4>
-<p><strong>最好的方案莫过于，复制报错代码，Google一下 </strong>
-下面提供几种，我安装时的几处坑</p>
+<h2 id="普通算数操作符">普通算数操作符</h2>
 <ul>
-<li>设置环境变量，注意一定是在caffe文件夹下，路径是caffe/python的绝对路径</li>
-<li>安装依赖报错，这个没什么好讲，直接google吧</li>
-<li>make的时候报错，想想看是否编辑了Makefile.config。之后make
-clean一下，再重新编译一下</li>
-<li>CPU_ONLY这个选项为1时，需要修改一下<code>caffe/examples/mnist/lenet_solver.prototxt</code>，将其中的<code>solver_mode:GPU</code>改为CPU</li>
+<li><code>__add__(self, other)</code>: 实现加法</li>
+<li><code>__sub__(self, other)</code>: 实现减法</li>
+<li><code>__mul__(self, other)</code>: 实现乘法</li>
+<li><code>__floordiv__(self, other)</code>: 实现整除法</li>
+<li><code>__div__(self, other)</code>: 实现除法</li>
+<li><code>__mod__(self, other)</code>: 实现取模</li>
+<li><code>__pow__(self, other)</code>: 实现指数</li>
+<li><code>__lshift__(self, other)</code>: 实现&lt;&lt;</li>
+<li><code>__rshift__(self, other)</code>: 实现&gt;&gt;</li>
+<li><code>__and__(self, other)</code>: 实现&amp;</li>
+<li><code>__or__(self, other)</code>: 实现|</li>
+<li><code>__xor__(self, other)</code>: 实现^</li>
 </ul>
-<p><a
-href="http://blog.csdn.net/sinat_26917383/article/details/53502719">参考文章</a></p>
-<h3 id="实现前准备">实现前准备</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 默认已安装git，未安装请sudo apt get install git</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/fzliu/style-transfer</span><br><span class="line"></span><br><span class="line"><span class="comment"># pycaffe环境布置</span></span><br><span class="line"><span class="built_in">sudo</span> pip install progressbar</span><br><span class="line"></span><br><span class="line"><span class="comment"># 下载训练模型，这里推荐vgg16</span></span><br><span class="line"><span class="comment"># 方法一，使用scripts/ 下的download_models.sh这个方法可能很慢。。。。</span></span><br><span class="line">bash scripts/download_models.sh vgg16</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">wget http://www.robots.ox.ac.uk/~vgg/software/very_deep/caffe/VGG_ILSVRC_16_layers.caffemodel</span><br></pre></td></tr></table></figure>
-<p>使用方法二下载的model需要放在./model/vgg16下，使用方法一下载的已经默认放在那</p>
-<h3 id="实现">实现</h3>
-<p>确保在源码文件夹中</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 格式</span></span><br><span class="line">python style.py -s &lt;style_image&gt; -c &lt;content_image&gt; -m &lt;model_name&gt; -g 0</span><br><span class="line"></span><br><span class="line"><span class="comment"># 举个栗子，model_name直接写vgg16或其他，不需要加具体路径</span></span><br><span class="line">python style.py -s images/style/starry_night.jpg -c images/content/nanjing.jpg -m vgg16 -g -1</span><br></pre></td></tr></table></figure>
-<p>提示：CPU很慢，真的很慢，估计要等一会才会有时间显示出来</p>
-<p>参数解析：</p>
+<h2 id="其他">其他</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">    <span class="variable language_">self</span>.name = value</span><br><span class="line">    <span class="comment">#每当属性被赋值的时候， ``__setattr__()`` 会被调用，这样就造成了递归调用。</span></span><br><span class="line">    <span class="comment">#这意味这会调用 ``self.__setattr__(&#x27;name&#x27;, value)`` ，每次方法会调用自己。这样会造成程序崩溃。</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">    <span class="variable language_">self</span>.__dict__[name] = value  <span class="comment">#给类中的属性名分配值</span></span><br><span class="line">    <span class="comment">#定制特有属性</span></span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">AccessCounter</span>:</span><br><span class="line">    <span class="string">&#x27;&#x27;&#x27;一个包含计数器的控制权限的类每当值被改变时计数器会加一&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, val</span>):</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="number">0</span>)</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;value&#x27;</span>, val)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__setattr__</span>(<span class="params">self, name, value</span>):</span><br><span class="line">        <span class="keyword">if</span> name == <span class="string">&#x27;value&#x27;</span>:</span><br><span class="line">            <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="variable language_">self</span>.counter + <span class="number">1</span>)</span><br><span class="line">    <span class="comment">#如果你不想让其他属性被访问的话，那么可以抛出 AttributeError(name) 异常</span></span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(name, value)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__delattr__</span>(<span class="params">self, name</span>):</span><br><span class="line">        <span class="keyword">if</span> name == <span class="string">&#x27;value&#x27;</span>:</span><br><span class="line">            <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__setattr__(<span class="string">&#x27;counter&#x27;</span>, <span class="variable language_">self</span>.counter + <span class="number">1</span>)</span><br><span class="line">        <span class="built_in">super</span>(AccessCounter, <span class="variable language_">self</span>).__delattr__(name)]</span><br></pre></td></tr></table></figure>
 <ul>
-<li>-s, 风格图位置；</li>
-<li>-c, 内容图位置；</li>
-<li>-m, 模型位置；</li>
-<li>-g, 什么模式，-1为CPU，0为单个GPU，1为两个GPU。</li>
-</ul>
-<h3 id="调整参数">调整参数</h3>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 确保在源码文件夹下,修改style.py配置文件</span></span><br><span class="line">vim style.py</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 修改文件大小，改为1024</span></span><br><span class="line">parser.add_argument(<span class="string">&quot;-l&quot;</span>, <span class="string">&quot;--length&quot;</span>, default=1024, <span class="built_in">type</span>=<span class="built_in">float</span>, required=False, <span class="built_in">help</span>=<span class="string">&quot;maximum image length&quot;</span>)</span><br><span class="line">def transfer_style(self, img_style, img_content, length=1024, ratio=1e5,</span><br><span class="line">n_iter=512, init=<span class="string">&quot;-1&quot;</span>, verbose=False, callback=None)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 修改迭代次数，个人认为400就差不多了，可自行修改；50次迭代之后改变的就是背景纹理，按需更改</span></span><br><span class="line">parser.add_argument(<span class="string">&quot;-n&quot;</span>, <span class="string">&quot;--num-iters&quot;</span>, default=400, <span class="built_in">type</span>=int, required=False, <span class="built_in">help</span>=<span class="string">&quot;L-BFGS iterations&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 还有很多其他可以修改的地方，在此就不一一赘述了</span></span><br></pre></td></tr></table></figure>
-<h2
-id="fast-neural-style-tensorflow的实现">fast-neural-style-tensorflow的实现</h2>
-<p><strong>环境介绍 </strong></p>
-<p>Ubuntu16.04 + CPU + python2.7 + tensorflow 1.0</p>
-<h3 id="搭建tensorflow环境">搭建tensorflow环境</h3>
-<p>环境说明：Ubuntu16.04 注：tensorflow只支持64位系统</p>
-<h4 id="安装前准备">安装前准备</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 确保有pip工具，一般自带，没有的话请按下面这条命令进行安装</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-pip python-dev</span><br></pre></td></tr></table></figure>
-<h4 id="cpu安装">CPU安装</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2</span></span><br><span class="line"><span class="built_in">sudo</span> pip installl tensorflow</span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line"><span class="built_in">sudo</span> pip3 install tensorflow</span><br></pre></td></tr></table></figure>
-<h4 id="gpu安装">GPU安装</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 安装地址请根据自己需求进行声明环境变量</span></span><br><span class="line"><span class="comment"># python2</span></span><br><span class="line"><span class="built_in">export</span> TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.12.0rc0-cp27-none-linux_x86_64.whl</span><br><span class="line"></span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line"><span class="built_in">export</span> TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.12.0rc0-cp35-cp35m-linux_x86_64.whl</span><br><span class="line"></span><br><span class="line"><span class="comment"># Python2</span></span><br><span class="line"><span class="built_in">sudo</span> pip install --upgrade <span class="variable">$TF_BINARY_URL</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># Python3</span></span><br><span class="line"><span class="built_in">sudo</span> pip3 install --upgrade <span class="variable">$TF_BINARY_URL</span></span><br></pre></td></tr></table></figure>
-<p><a
-href="https://morvanzhou.github.io/tutorials/machine-learning/tensorflow/1-2-install/">参考文章</a></p>
-<h4 id="源码安装">源码安装</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载最新的tensorflow项目</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/tensorflow/tensorflow</span><br><span class="line"><span class="comment"># 进入tensorflow</span></span><br><span class="line"><span class="built_in">cd</span> tensorflow</span><br><span class="line"><span class="comment"># 切换到需要安装的版本分支，以1.0为栗子</span></span><br><span class="line">git checkout r1.0</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装bazel</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&quot;deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8&quot;</span> | <span class="built_in">sudo</span> <span class="built_in">tee</span> /etc/apt/sources.list.d/bazel.list</span><br><span class="line"></span><br><span class="line">curl https://bazel.build/bazel-release.pub.gpg | <span class="built_in">sudo</span> apt-key add -</span><br><span class="line"></span><br><span class="line"><span class="built_in">sudo</span> apt-get update &amp;&amp; <span class="built_in">sudo</span> apt-get install bazel</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装依赖</span></span><br><span class="line"><span class="comment"># Python 2</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-numpy python-dev python-pip python-wheel</span><br><span class="line"></span><br><span class="line"><span class="comment"># Python 3</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python3-numpy python3-dev python3-pip python3-wheel</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行编译，确保在tensorflow文件夹中</span></span><br><span class="line">./configure</span><br><span class="line"><span class="comment"># 提示信息可以一路回车</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译CPU版本</span></span><br><span class="line">bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package</span><br><span class="line"><span class="comment"># 编译GPU版本</span></span><br><span class="line">bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译生成的whl包</span></span><br><span class="line">bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装生成的包</span></span><br><span class="line"><span class="built_in">sudo</span> pip install /tmp/tensorflow_pkg/tensorflow*.whl</span><br></pre></td></tr></table></figure>
-<p><a
-href="https://tensorflow.feisky.xyz/install/src.html">参考文章</a></p>
-<h4 id="测试-1">测试</h4>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 打开对应版本的python终端</span></span><br><span class="line">&gt;import tensorflow</span><br><span class="line"><span class="comment"># 未报错即安装成功</span></span><br></pre></td></tr></table></figure>
-<h3 id="实现前准备-1">实现前准备</h3>
-<p>下载已经训练好的模型</p>
-<p><a
-href="https://pan.baidu.com/s/1i4GTS4d#list/path=%2F">百度云盘链接</a></p>
-<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line"># 安装pyyaml</span><br><span class="line">sudo pip install pyyaml</span><br></pre></td></tr></table></figure>
-<h3 id="实现-1">实现</h3>
-<p>确保在源码文件夹中</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 格式</span></span><br><span class="line">python eval.py --model_file &lt;path of ckpt-done&gt; --image_file &lt;path of image&gt;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 举个栗子,假设wave.ckpt-done放在了源码文件夹中的models文件夹中</span></span><br><span class="line">python eval.py --model_file models/wave.ckpt-done --image_file img/test.jpg</span><br><span class="line"><span class="comment"># 默认生成位置在generated，默认生成文件名称为res.jpg。可自行在eval.py中更改</span></span><br></pre></td></tr></table></figure>
-<h3 id="训练新的风格模型">训练新的风格模型</h3>
-<p><strong>下载</strong></p>
-<p><a
-href="http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz">vgg16训练模型</a>(500多M)</p>
-<p><a
-href="http://msvocds.blob.core.windows.net/coco2014/train2014.zip">训练数据集</a>(12.6G)</p>
-<p>或者使用wget命令下载</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># vgg16训练模型</span></span><br><span class="line">wget http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练数据集</span></span><br><span class="line">wget http://msvocds.blob.core.windows.net/coco2014/train2014.zip</span><br></pre></td></tr></table></figure>
-<p><strong>移动模型到对应目录中</strong></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在源码文件夹中新建文件夹</span></span><br><span class="line"><span class="built_in">mkdir</span> pretrained</span><br><span class="line"></span><br><span class="line"><span class="comment"># 下载完成之后需要进行解压</span></span><br><span class="line">tar -zxvf vgg_16_2016_08_28.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 移动文件夹到pretrained</span></span><br><span class="line"><span class="built_in">cp</span> &lt;path of vgg16&gt; &lt;pretrained&gt;</span><br></pre></td></tr></table></figure>
-<p><strong>训练新的模型</strong></p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压数据集</span></span><br><span class="line">unzip train2014.zip</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入数据集</span></span><br><span class="line"><span class="built_in">cd</span> train2014</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建立软链接，方便训练命令的输入</span></span><br><span class="line"><span class="built_in">ln</span> -s &lt;train2014的绝对路径&gt; train2014</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入源码文件夹中</span></span><br><span class="line"><span class="comment"># 开始训练, wave.yml是作者预先配置好的文件，如果训练自己的模型，需要自己新写一份yml文件</span></span><br><span class="line">python train.py -c conf/wave.yml</span><br></pre></td></tr></table></figure>
-<h2 id="总结">总结</h2>
-<p>如果只是单纯的想玩玩风格转换的话，可以试试fast-neural-style-tensorflow，不过训练新模型还是不建议（特殊需要除外），耗时耗力</p>
-<p>如果想锻炼自己，从零开始学习，可以从读论文开始，然后试试style-transfer代码</p>
+<li><p><code>__str__(self)</code>: 返回值，人类可读</p></li>
+<li><p><code>__repr__(self)</code>: 返回值，机器可读</p></li>
+<li><p><code>__unicode__(self)</code>: 返回unicode()</p></li>
+<li><p><code>__hash__(self)</code>: hash()返回一个整形</p></li>
+<li><p><code>__nonzero__(self)</code>: bool()返回的值</p></li>
+<li><p><code>__getattr__(self, name)</code>:
+当用户试图获取一个不存在（或不建议的属性）的属性时，给出一些警告</p></li>
+<li><p><code>__setattr__(self, name, value)</code>:
+无论属性是否存在，都允许你定义对属性的赋值行为</p></li>
+<li><p><code>__delattr__(self, name)</code>： 删除一个属性</p></li>
+</ul>
+<h2 id="封装附加魔术方法">封装附加魔术方法</h2>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">FunctionalList</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;一个封装了一些附加魔术方法比如 head, tail, init, last, drop, 和take的列表类。</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, values=<span class="literal">None</span></span>):</span><br><span class="line">        <span class="keyword">if</span> values <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            <span class="variable language_">self</span>.values = []</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="variable language_">self</span>.values = values</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__len__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">len</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__getitem__</span>(<span class="params">self, key</span>):</span><br><span class="line">        <span class="comment">#如果键的类型或者值无效，列表值将会抛出错误</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[key]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__setitem__</span>(<span class="params">self, key, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>.values[key] = value</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__delitem__</span>(<span class="params">self, key</span>):</span><br><span class="line">        <span class="keyword">del</span> <span class="variable language_">self</span>.values[key]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__iter__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">iter</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__reversed__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">reversed</span>(<span class="variable language_">self</span>.values)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">append</span>(<span class="params">self, value</span>):</span><br><span class="line">        <span class="variable language_">self</span>.values.append(value)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">head</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">tail</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[<span class="number">1</span>:]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">init</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="comment">#返回一直到末尾的所有元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[:-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">last</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="comment">#返回末尾元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">drop</span>(<span class="params">self, n</span>):</span><br><span class="line">        <span class="comment">#返回除前n个外的所有元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[n:]</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">take</span>(<span class="params">self, n</span>):</span><br><span class="line">        <span class="comment">#返回前n个元素</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.values[:n]</span><br></pre></td></tr></table></figure>
+<h2 id="反射">反射</h2>
+<ul>
+<li><code>__instancecheck__(self, instance)</code>:
+检查一个实例是否为定义类的实例</li>
+<li><code>__subclasscheck__(self, subclass)</code>:
+检查一个类是否为定义类的子类</li>
+</ul>
+<h2 id="更改状态">更改状态</h2>
+<ul>
+<li><code>__call__(self, [...])</code>:
+经常改变状态的时候，调用这个方法是一种直接和优雅的方法</li>
+</ul>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Entity</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;调用实体来改变实体的位置。&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, size, x, y</span>):</span><br><span class="line">        <span class="variable language_">self</span>.x, <span class="variable language_">self</span>.y = x, y</span><br><span class="line">        <span class="variable language_">self</span>.size = size</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__call__</span>(<span class="params">self, x, y</span>):</span><br><span class="line">        <span class="string">&#x27;&#x27;&#x27;改变实体的位置&#x27;&#x27;&#x27;</span></span><br><span class="line">        <span class="variable language_">self</span>.x, <span class="variable language_">self</span>.y = x, y</span><br></pre></td></tr></table></figure>
+<h2 id="会话管理">会话管理</h2>
+<p>使用with语句来调用</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">class</span> <span class="title class_">Closer</span>:</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;通过with语句和一个close方法来关闭一个对象的会话管理器&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, obj</span>):</span><br><span class="line">        <span class="variable language_">self</span>.obj = obj</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__enter__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.obj <span class="comment"># bound to target</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__exit__</span>(<span class="params">self, exception_type, exception_val, trace</span>):</span><br><span class="line">        <span class="keyword">try</span>:</span><br><span class="line">            <span class="variable language_">self</span>.obj.close()</span><br><span class="line">        <span class="keyword">except</span> AttributeError: <span class="comment"># obj isn&#x27;t closable</span></span><br><span class="line">            <span class="built_in">print</span>(<span class="string">&#x27;Not closable.&#x27;</span>)</span><br><span class="line">        <span class="keyword">return</span> <span class="literal">True</span> <span class="comment"># exception handled successfully</span></span><br></pre></td></tr></table></figure>
+<h2 id="slots__"><code>__slots__</code></h2>
+<p>限制class的属性，但不限制class的子类的属性，除非子类也使用<code>__slots__</code></p>
+<p>例子：</p>
+<p>使用<code>__slots__</code>会比不使用，减少40%~50%的内存</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 不使用__slots__</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyClass</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">  <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, name, identifier</span>):</span><br><span class="line">      <span class="variable language_">self</span>.name = name</span><br><span class="line">      <span class="variable language_">self</span>.identifier = identifier</span><br><span class="line">      <span class="variable language_">self</span>.set_up()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用__slots__</span></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">MyClass</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">  __slots__ = [<span class="string">&#x27;name&#x27;</span>, <span class="string">&#x27;identifier&#x27;</span>]</span><br><span class="line">  <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, name, identifier</span>):</span><br><span class="line">      <span class="variable language_">self</span>.name = name</span><br><span class="line">      <span class="variable language_">self</span>.identifier = identifier</span><br><span class="line">      <span class="variable language_">self</span>.set_up()</span><br></pre></td></tr></table></figure>
+]]></content>
+      <categories>
+        <category>Python</category>
+      </categories>
+      <tags>
+        <tag>Magic Methods</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>crontab命令</title>
+    <url>/hexo_blog/2017/08/24/%E8%BF%90%E7%BB%B4/crontab%E5%91%BD%E4%BB%A4/</url>
+    <content><![CDATA[<p>学会使用crontab定时任务，提高工作效率。详解了crontab的基本格式和例子，以及相关目录介绍和日志文件的设置。</p>
+<span id="more"></span>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看当前用户的定时任务，也可以使用crontab -uroot -l查看指定用户的定时任务</span></span><br><span class="line">crontab -l</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编辑crontab定时任务，貌似是使用vim的编辑命令</span></span><br><span class="line">crontab -e</span><br><span class="line"></span><br><span class="line"><span class="comment"># 其实基本上知道这两个就可以处理定时任务了</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 删除当前用户的所有定时任务，不知道为什么会有这种设定（手贱按下过一次，恢复起来真的惨）</span></span><br><span class="line">crontab -r</span><br><span class="line"><span class="comment"># 如果按到了，就去日志文件里面找记录，尽可能恢复命令吧</span></span><br></pre></td></tr></table></figure>
+<p>下面讲讲编辑任务的格式</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 基本格式</span></span><br><span class="line"></span><br><span class="line">  *  *  *  *  * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 分 时 日 月 周 需要执行的命令</span></span><br></pre></td></tr></table></figure>
+<p>可视化测试在线网站，https://crontab.guru/</p>
+<p>几个栗子</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 每分钟执行一次command命令，空格表示间隔</span></span><br><span class="line">* * * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每小时执行一次，‘/’表示频率</span></span><br><span class="line">0 */1 * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每小时的10分和40分各执行一次，‘,’表示并列</span></span><br><span class="line">10,40 * * * * <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每个星期一的10-14点的10，40执行，‘-’表述区间</span></span><br><span class="line">10,40 10-14 * * 1 <span class="built_in">command</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 每天的8，13，20点执行，注意此时表示分的那个*要替换为0</span></span><br><span class="line">0 8,13,20 * * * <span class="built_in">command</span></span><br></pre></td></tr></table></figure>
+<p>日志文件一般存放在
+<code>/var/log/cron*</code>中，不过我的Ubuntu好像没有开启cron日志，可以在
+<code>/etc/rsyslog.d/50-default.conf</code>中进行设置（Centos在
+<code>/etc/rsyslog.conf</code>中设置）</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">&gt; <span class="built_in">sudo</span> vim /etc/rsyslog.d/50-default.conf</span><br><span class="line"><span class="comment"># 找到cron*, 去掉前面的#注释即可</span></span><br><span class="line"><span class="comment"># 重启rsyslog</span></span><br><span class="line">&gt; <span class="built_in">sudo</span> service rsyslog  restart</span><br></pre></td></tr></table></figure>
+<p>crontab相关目录介绍</p>
+<p><code>/etc/anacrontab</code>:
+这个文件存着系统级的任务。它主要用来运行每日的(daily)，每周的(weekly),每月的(monthly)的任务。一般不在此文件安装自己的任务</p>
+<p><code>/etc/cron.d/</code>: 此目录下存放的是系统级任务的任务文件。</p>
+<p><code>/var/spool/cron/</code>:
+此目录下存放各个用户的任务文件。各个用户的任务存放在以自已用户名为文件名的任务文件中。此文件中的指令行没有用户域。</p>
+<p><code>/etc/crontab</code>:
+crontab的主要配置文件，执行了每小时、每日、每周、每月的脚本文件</p>
+<p><code>/etc/cron.hourly</code>、<code>/etc/cron.daily</code>、<code>/etc/cron.weekly</code>、<code>/etc/cron.monthly</code>：这四个目录下分别对应着每隔一段时间需要执行的脚本任务，脚本使用shell命令编写</p>
+<p><a href="https://segmentfault.com/a/1190000002628040"></a></p>
 <p><a
-href="http://blog.csdn.net/sinat_26917383/article/details/53978519">style-transfer
-参考文章</a></p>
-<p><a href="https://github.com/fzliu/style-transfer">style-transfer
-代码</a></p>
+href="http://www.cnblogs.com/peida/archive/2013/01/08/2850483.html">每天一个linux命令（50）：crontab命令</a></p>
+<p><a href="https://segmentfault.com/a/1190000002628040">Linux
+下执行定时任务 crontab 命令详解</a></p>
+]]></content>
+      <categories>
+        <category>Software</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>crontab</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>SQL语句的优化</title>
+    <url>/hexo_blog/2018/04/07/%E6%95%B0%E6%8D%AE%E5%BA%93/SQL%E8%AF%AD%E5%8F%A5%E7%9A%84%E4%BC%98%E5%8C%96/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了关于SQL语句（主要是Oracle数据库）的优化方案。
+<span id="more"></span></p>
 <p><a
-href="https://github.com/hzy46/fast-neural-style-tensorflow/blob/master/README.md#use-trained-models">fast-neural-style-tensorflow
-代码</a></p>
+href="https://juejin.im/post/5ab0f39df265da23866fba55">数据库性能优化之SQL语句优化</a></p>
+<h2 id="操作符优化">操作符优化</h2>
+<h3 id="in-操作符">IN 操作符</h3>
+<p>优点：编写容易，清晰易懂</p>
+<p>缺点：性能较低。</p>
+<p>Oracle执行步骤：</p>
+<ol type="1">
+<li>将其转换多表连接</li>
+<li>如果转换不成功则先执行IN里面的子查询，再执行外层表记录</li>
+<li>如果转换成功则直接采用多表连接查询</li>
+</ol>
+<p>所以用IN的SQL至少会多一个转换过程。</p>
+<p>替代方案：在业务密集的SQL用EXISTS替代</p>
+<h3 id="not-in操作符">NOT IN操作符</h3>
+<p>不能应用表的索引，不建议使用。</p>
+<p>替代方案：用NOT EXISTS方案</p>
+<h3 id="is-null或is-not-null操作">IS NULL或IS NOT NULL操作</h3>
+<p>判断字段为空一般是不会用索引的（索引不索引空值）。如果某列存在空值，即使对该列建索引也不会提高性能。</p>
+<p>替代方案：将<code>a is not null</code>改为<code>a&gt;0</code>或<code>a&gt;' '</code>等。不允许字段为空，而用一个缺省值代替空值。</p>
+<h3 id="及-操作符">&lt; 及 &gt; 操作符</h3>
+<p>一般情况下是不需要调整的，在某些情况下可以进行优化。</p>
+<p>如：一个表有100万记录，一个数值型字段A，有30万的A=0，39万A=2，
+1万A=3。在这种情况下执行A&gt;2与A&gt;=3有很大区别，<code>A&gt;2</code>在Oracle中会先找出为2的记录索引再进行比较，但是<code>A&gt;=3</code>会直接找到等于3的记录索引。</p>
+<h3 id="like操作符">LIKE操作符</h3>
+<p>LIKE医用通配符查询，基本上可以达到任一查询。但是不同语句可能会产生性能上的很大差异。如<code>LIKE '%5400%'</code>这种情况下不会引用索引，<code>LIKE 'X5400%'</code>会引用范围索引。即前者会产生全表扫描，后者会进行范围内的查询，后者相对于前者的性能大大提高了。</p>
+<h3 id="union操作符">UNION操作符</h3>
+<p>UNION在进行表连接之后会筛选掉重复的记录，所以在表连接之后会对产生的结果集进行排序运算，删除重复记录再返回结果。但是实际上大部分应用中是不会产生重复的记录，最常见的是过程表于历史表UNION。在这种情况下，替代方案就是使用UNION
+ALL代替UNION。UNION ALL操作只是简单的将两个结果合并后就返回</p>
+<h3 id="联接列">联接列</h3>
+<p>对于有联接的列，优化器是不会使用索引的。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 使用联接查询</span><br><span class="line"><span class="keyword">select</span> <span class="operator">*</span> <span class="keyword">from</span> employss <span class="keyword">where</span> first_name<span class="operator">||</span><span class="string">&#x27;&#x27;</span><span class="operator">||</span>last_name <span class="operator">=</span><span class="string">&#x27;Beill Cliton&#x27;</span>;</span><br><span class="line"># 不使用联接查询，基于last_name创建索引</span><br><span class="line"><span class="keyword">where</span> first_name <span class="operator">=</span><span class="string">&#x27;Beill&#x27;</span> <span class="keyword">and</span> last_name <span class="operator">=</span><span class="string">&#x27;Cliton&#x27;</span>;</span><br></pre></td></tr></table></figure>
+<h3 id="order-by语句">Order By语句</h3>
+<p>将返回的查询结果进行排序。任何在Order
+by语句的非索引项或者有计算表达式都将降低查询速度。解决方案就是为所使用的列建立另一个索引，同时也绝对避免在order
+by子句中使用表达式。</p>
+<h3 id="not-语句">NOT 语句</h3>
+<p>NOT可用来对任何逻辑运算符号取反。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 如果要使用<span class="keyword">NOT</span>，则应在取反的短语前面机上括号，并在短语前面机上<span class="keyword">NOT</span>运算符。</span><br><span class="line"><span class="keyword">where</span> <span class="keyword">not</span> (status <span class="operator">=</span> <span class="string">&#x27;VALID&#x27;</span>)</span><br><span class="line"># <span class="keyword">NOT</span>运算符包含在另外一个逻辑运算符<span class="operator">&lt;&gt;</span>运算符中。即使不在查询的<span class="keyword">where</span>子句中显式地加入<span class="keyword">NOT</span>词，<span class="keyword">NOT</span>仍在运算符中</span><br><span class="line"><span class="keyword">where</span> status <span class="operator">&lt;&gt;</span> <span class="string">&#x27;INVALID&#x27;</span>;</span><br><span class="line"># 这种情况运行建立salary的索引，在速度上会优于上面的方案</span><br><span class="line"><span class="keyword">select</span> <span class="operator">*</span> <span class="keyword">from</span> employee <span class="keyword">where</span> salary <span class="operator">&lt;</span> <span class="number">3000</span> <span class="keyword">or</span> salary <span class="operator">&gt;</span> <span class="number">3000</span>;</span><br></pre></td></tr></table></figure>
+<h2 id="书写的影响">书写的影响</h2>
+<h3
+id="同一功能同一性能不同写法sql的影响">同一功能同一性能不同写法SQL的影响</h3>
+<p>尽可能避免不同写法的出现，根据Oracle共享内存SGA原理，如果语句不同则进行分析，占用共享内存。如果SQL字符串及格式完全相同，则Oracle只会分析一次，共享内存也只会留下一次的分析结果</p>
+<h3 id="where后面的条件顺序影响">WHERE后面的条件顺序影响</h3>
+<p>下面两个语句，在没有建立索引的情况下，执行都是全表扫描。</p>
+<p><code>dy_dj = '1KV以下'</code>这个条件在记录集内比率为99%,
+<code>xh_bz=1</code>而这个条件只有0.5%。所以第二条SQL的CPU占用率明显低于第一条。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">Select</span> <span class="operator">*</span> <span class="keyword">from</span> zl_yhjbqk <span class="keyword">where</span> dy_dj <span class="operator">=</span> <span class="string">&#x27;1KV以下&#x27;</span> <span class="keyword">and</span> xh_bz<span class="operator">=</span><span class="number">1</span></span><br><span class="line"><span class="keyword">Select</span> <span class="operator">*</span> <span class="keyword">from</span> zl_yhjbqk <span class="keyword">where</span> xh_bz<span class="operator">=</span><span class="number">1</span> <span class="keyword">and</span> dy_dj <span class="operator">=</span> <span class="string">&#x27;1KV以下&#x27;</span></span><br></pre></td></tr></table></figure>
+<h3 id="查询表顺序的影响">查询表顺序的影响</h3>
+<p>在FROM后面的表中的列表顺序会对SQL执行性能影响，在没有索引及ORACLE没有对表进行统计分析的情况下，ORACLE会按<strong>表出现的顺序</strong>进行连接，所以如果表的顺序不对时会产生十分消耗服务器资源的数据交叉（如果对表进行了统计分析，Oracle自动先进小表的连接，再进行大表的连接）</p>
+<h2 id="sql语句索引的利用">SQL语句索引的利用</h2>
+<h3 id="条件字段的优化">条件字段的优化</h3>
+<p>函数处理的字段不能利用索引</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：substr(hbs_bh,<span class="number">1</span>,<span class="number">4</span>)<span class="operator">=</span><span class="string">&#x27;5400&#x27;</span></span><br><span class="line">优化处理：hbs_bh <span class="keyword">like</span> <span class="string">&#x27;5400%&#x27;</span></span><br><span class="line"></span><br><span class="line">原语句：trunc(sk_rq)<span class="operator">=</span>trunc(sysdate)</span><br><span class="line">优化处理：sk_rq<span class="operator">&gt;=</span>trunc(sysdate) <span class="keyword">and</span> sk_rq<span class="operator">&lt;</span>trunc(sysdate<span class="operator">+</span><span class="number">1</span>)</span><br></pre></td></tr></table></figure>
+<p>进行了显示或隐式的运算的字段不能进行索引。</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：<span class="string">&#x27;X&#x27;</span> <span class="operator">||</span> hbs_bh<span class="operator">&gt;</span><span class="string">&#x27;X5400021452&#x27;</span></span><br><span class="line">优化处理：hbs_bh<span class="operator">&gt;</span><span class="string">&#x27;5400021542&#x27;</span></span><br><span class="line"></span><br><span class="line">原语句：sk_rq<span class="operator">+</span><span class="number">5</span><span class="operator">=</span>sysdate</span><br><span class="line">优化处理：sk_rq<span class="operator">=</span>sysdate<span class="number">-5</span></span><br></pre></td></tr></table></figure>
+<p>条件内包括的多个本表的字段运算时不能进行索引</p>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">原语句：ys_df<span class="operator">&gt;</span>cx_df</span><br><span class="line">无法进行优化</span><br><span class="line"></span><br><span class="line">原语句：qc_bh <span class="operator">||</span> kh_bh<span class="operator">=</span><span class="string">&#x27;5400250000&#x27;</span></span><br><span class="line">优化处理：qc_bh<span class="operator">=</span><span class="string">&#x27;5400&#x27;</span> <span class="keyword">and</span> kh_bh<span class="operator">=</span><span class="string">&#x27;250000&#x27;</span></span><br></pre></td></tr></table></figure>
+<h2 id="其他优化方法">其他优化方法</h2>
+<ol type="1">
+<li><p>选择最有效率的表名顺序（只在基于规则的优化器中有效）</p>
+<p><code>ORACLE</code>
+的解析器按照从右到左的顺序处理FROM子句中的表名，FROM子句中写在最后的表(基础表
+driving
+table)将被最先处理，在FROM子句中包含多个表的情况下,你必须选择记录条数最少的表作为基础表。如果有3个以上的表连接查询,
+那就需要选择交叉表(intersection table)作为基础表,
+交叉表是指那个被其他表所引用的表</p></li>
+<li><p><code>WHERE</code>子句中的连接顺序</p>
+<p><code>ORACLE</code>采用自下而上的顺序解析WHERE子句,根据这个原理,表之间的连接必须写在其他<code>WHERE</code>条件之前,
+那些可以过滤掉最大数量记录的条件必须写在WHERE子句的末尾.</p></li>
+<li><p><code>SELECT</code>子句中避免使用 '*'</p>
+<p><code>ORACLE</code>在解析的过程中, 会将'*'依次转换成所有的列名,
+这个工作是通过查询数据字典完成的, 这意味着将耗费更多的时间。</p></li>
+<li><p>减少访问数据库的次数</p>
+<p><code>ORACLE</code>在内部执行了许多工作: 解析SQL语句,
+估算索引的利用率, 绑定变量 , 读数据块等。</p></li>
+<li><p>在<code>SQL*Plus</code> ,
+<code>SQL*Forms</code>和<code>Pro*C</code>中重新设置<code>ARRAYSIZE</code>参数,
+可以增加每次数据库访问的检索数据量 ,建议值为200</p></li>
+<li><p>使用<code>DECODE</code>函数来减少处理时间</p>
+<p>使用<code>DECODE</code>函数可以避免重复扫描相同记录或重复连接相同的表.</p></li>
+<li><p>整合简单,无关联的数据库访问</p></li>
+</ol>
+<p>如果你有几个简单的数据库查询语句,你可以把它们整合到一个查询中(即使它们之间没有关系)
+。</p>
+<ol start="8" type="1">
+<li>删除重复记录：</li>
+</ol>
+<blockquote>
+<p>最高效的删除重复记录方法(因为使用了<code>ROWID</code>)</p>
+</blockquote>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">DELETE</span> <span class="keyword">FROM</span> EMP E <span class="keyword">WHERE</span> E.ROWID <span class="operator">&gt;</span> (<span class="keyword">SELECT</span> <span class="built_in">MIN</span>(X.ROWID) <span class="keyword">FROM</span> EMP X <span class="keyword">WHERE</span> X.EMP_NO <span class="operator">=</span> E.EMP_NO)</span><br></pre></td></tr></table></figure>
+<ol start="9" type="1">
+<li><p>用<code>TRUNCATE</code>替代<code>DELETE</code></p>
+<p>当删除表中的记录时,在通常情况下, 回滚段(rollback segments )
+用来存放可以被恢复的信息.
+如果你没有<code>COMMIT</code>事务,<code>ORACLE</code>会将数据恢复到删除之前的状态(准确地说是恢复到执行删除命令之前的状况)
+而当运用<code>TRUNCATE</code>时,
+回滚段不再存放任何可被恢复的信息.当命令运行后,数据不能被恢复.因此很少的资源被调用,执行时间也会很短.
+(译者按:
+<code>TRUNCATE</code>只在删除全表适用,<code>TRUNCATE</code>是DDL不是DML)
+。</p></li>
+<li><p>尽量多使用<code>COMMIT</code></p>
+<p>只要有可能,在程序中尽量多使用<code>COMMIT</code>,
+这样程序的性能得到提高,需求也会因为<code>COMMIT</code>所释放的资源而减少，<code>COMMIT</code>所释放的资源:</p>
+<ol type="a">
+<li><p>回滚段上用于恢复数据的信息.</p></li>
+<li><p>被程序语句获得的锁</p></li>
+<li><p>redo log buffer 中的空间</p></li>
+<li><p><code>ORACLE</code>为管理上述3种资源中的内部花费</p></li>
+</ol></li>
+<li><p>用<code>Where</code>子句替换<code>HAVING</code>子句</p></li>
+</ol>
+<pre><code>避免使用`HAVING`子句, `HAVING` 只会在检索出所有记录之后才对结果集进行过滤. 这个处理需要排序,总计等操作. 如果能通过`WHERE`子句限制记录的数目,那就能减少这方面的开销. (非`oracle`中)`on`、`where`、`having`这三个都可以加条件的子句中，`on`是最先执行，`where`次之，`having`最后，因为`on`是先把不符合条件的记录过滤后才进行统计，它就可以减少中间运算要处理的数据，按理说应该速度是最快的，`where`也应该比`having`快点的，因为它过滤数据后才进行`sum`，在两个表联接时才用`on`的，所以在一个表的时候，就剩下`where`跟`having`比较了。在这单表查询统计的情况下，如果要过滤的条件没有涉及到要计算字段，那它们的结果是一样的，只是`where`可以使用`rushmore`技术，而`having`就不能，在速度上后者要慢如果要涉及到计算的字 段，就表示在没计算之前，这个字段的值是不确定的，根据上篇写的工作流程，`where`的作用时间是在计算之前就完成的，而`having`就是在计算后才起作 用的，所以在这种情况下，两者的结果会不同。在多表联接查询时，`on`比`where`更早起作用。系统首先根据各个表之间的联接条件，把多个表合成一个临时表 后，再由`where`进行过滤，然后再计算，计算完后再由`having`进行过滤。由此可见，要想过滤条件起到正确的作用，首先要明白这个条件应该在什么时候起作用，然后再决定放在那里。</code></pre>
+<ol start="12" type="1">
+<li>减少对表的查询</li>
+</ol>
+<blockquote>
+<p>在含有子查询的SQL语句中,要特别注意减少对表的查询</p>
+</blockquote>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">SELECT</span> TAB_NAME <span class="keyword">FROM</span> TABLES <span class="keyword">WHERE</span> (TAB_NAME,DB_VER) <span class="operator">=</span> ( <span class="keyword">SELECT</span> TAB_NAME,DB_VER <span class="keyword">FROM</span> TAB_COLUMNS <span class="keyword">WHERE</span> VERSION <span class="operator">=</span> <span class="number">604</span>)</span><br></pre></td></tr></table></figure>
+<ol start="13" type="1">
+<li><p>通过内部函数提高SQL效率</p>
+<p>复杂的SQL往往牺牲了执行效率.
+能够掌握上面的运用函数解决问题的方法在实际工作中是非常有意义的。</p></li>
+<li><p>使用表的别名(<code>Alias</code>)：</p></li>
+</ol>
+<pre><code>当在SQL语句中连接多个表时, 请使用表的别名并把别名前缀于每个Column上.这样一来,就可以减少解析的时间并减少那些由Column歧义引起的语法错误。</code></pre>
+<ol start="15" type="1">
+<li>识别’低效执行’的SQL语句</li>
+</ol>
+<pre><code>虽然目前各种关于SQL优化的图形化工具层出不穷,但是写出自己的SQL工具来解决问题始终是一个最好的方法。</code></pre>
+<p>​<code>sql SELECT EXECUTIONS, DISK_READS, BUFFER_GETS, ROUND((BUFFER_GETS-DISK_READS)/BUFFER_GETS,2) Hit_radio, ROUND(DISK_READS/EXECUTIONS,2) Reads_per_run, SQL_TEXT FROM V$SQLAREA WHERE EXECUTIONS&gt;0 AND BUFFER_GETS &gt; 0 AND (BUFFER_GETS-DISK_READS)/BUFFER_GETS &lt; 0.8 ORDER BY 4 DESC; ​</code></p>
+<ol start="16" type="1">
+<li><p>用索引提高效率</p>
+<p>索引是表的一个概念部分,用来提高检索数据的效率，ORACLE使用了一个复杂的自平衡B-tree结构.
+通常,通过索引查询数据比全表扫描要快.
+当ORACLE找出执行查询和Update语句的最佳路径时, ORACLE优化器将使用索引.
+同样在联结多个表时使用索引也可以提高效率.
+另一个使用索引的好处是,它提供了主键(primary
+key)的唯一性验证.。那些LONG或LONG RAW数据类型, 你可以索引几乎所有的列.
+通常, 在大型表中使用索引特别有效. 当然,你也会发现,
+在扫描小表时,使用索引同样能提高效率.
+虽然使用索引能得到查询效率的提高,但是我们也必须注意到它的代价.
+索引需要空间来存储,也需要定期维护, 每当有记录在表中增减或索引列被修改时,
+索引本身也会被修改. 这意味着每条记录的INSERT , DELETE ,
+UPDATE将为此多付出4 , 5 次的磁盘I/O .
+因为索引需要额外的存储空间和处理,那些不必要的索引反而会使查询反应时间变慢.。定期的重构索引是有必要的</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="keyword">ALTER</span> INDEX <span class="operator">&lt;</span>INDEXNAME<span class="operator">&gt;</span> REBUILD <span class="operator">&lt;</span>TABLESPACENAME<span class="operator">&gt;</span></span><br></pre></td></tr></table></figure>
+<ol start="17" type="1">
+<li><p>用<code>EXISTS</code>替换<code>DISTINCT</code></p>
+<p>当提交一个包含一对多表信息(比如部门表和雇员表)的查询时,避免在<code>SELECT</code>子句中使用<code>DISTINCT</code>.
+一般可以考虑用<code>EXIST</code>替换, <code>EXISTS</code>
+使查询更为迅速,因为RDBMS核心模块将在子查询的条件一旦满足后,立刻返回结果.
+例子：</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">(低效):</span><br><span class="line"><span class="keyword">SELECT</span> <span class="keyword">DISTINCT</span> DEPT_NO, DEPT_NAME <span class="keyword">FROM</span> DEPT D, EMP E <span class="keyword">WHERE</span> D.DEPT_NO <span class="operator">=</span> E.DEPT_NO</span><br><span class="line">(高效):</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_NO, DEPT_NAME <span class="keyword">FROM</span> DEPT D <span class="keyword">WHERE</span> <span class="keyword">EXISTS</span> ( <span class="keyword">SELECT</span> <span class="string">&#x27;X&#x27;</span> <span class="keyword">FROM</span> EMP E <span class="keyword">WHERE</span> E.DEPT_NO <span class="operator">=</span> D.DEPT_NO);</span><br></pre></td></tr></table></figure>
+<ol start="18" type="1">
+<li><p>sql语句用大写的</p>
+<p>因为oracle总是先解析sql语句，把小写的字母转换成大写的再执行</p></li>
+<li><p>在java代码中尽量少用连接符“＋”连接字符串！</p></li>
+<li><p>避免在索引列上使用NOT，通常我们要避免在索引列上使用NOT,
+NOT会产生在和在索引列上使用函数相同的影响.
+当ORACLE”遇到”NOT,他就会停止使用索引转而执行全表扫描。</p></li>
+<li><p>避免在索引列上使用计算</p>
+<p>WHERE子句中，如果索引列是函数的一部分．优化器将不使用索引而使用全表扫描．</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效：</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> SAL <span class="operator">*</span> <span class="number">12</span> <span class="operator">&gt;</span> <span class="number">25000</span>;</span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> SAL <span class="operator">&gt;</span> <span class="number">25000</span><span class="operator">/</span><span class="number">12</span>;</span><br></pre></td></tr></table></figure>
+<ol start="22" type="1">
+<li>用&gt;=替代&gt;</li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> DEPTNO <span class="operator">&gt;=</span> <span class="number">4</span></span><br><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> DEPTNO <span class="operator">&gt;</span> <span class="number">3</span></span><br></pre></td></tr></table></figure>
+<ol start="23" type="1">
+<li><p>用UNION替换OR (适用于索引列)</p>
+<p>通常情况下, 用UNION替换WHERE子句中的OR将会起到较好的效果.
+对索引列使用OR将造成全表扫描. 注意, 以上规则只针对多个索引列有效.
+如果有column没有被索引, 查询效率可能会因为你没有选择OR而降低.
+在下面的例子中, LOC_ID 和REGION上都建有索引.</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span></span><br><span class="line"><span class="keyword">UNION</span></span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> REGION <span class="operator">=</span> &quot;MELBOURNE&quot;</span><br><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> LOC_ID , LOC_DESC , REGION</span><br><span class="line"><span class="keyword">FROM</span> LOCATION</span><br><span class="line"><span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span> <span class="keyword">OR</span> REGION <span class="operator">=</span> &quot;MELBOURNE&quot;</span><br></pre></td></tr></table></figure>
+<ol start="24" type="1">
+<li><p>用IN来替换OR</p>
+<p>这是一条简单易记的规则，但是实际的执行效果还须检验，在ORACLE8下，两者的执行路径似乎是相同的．</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span>…. <span class="keyword">FROM</span> LOCATION <span class="keyword">WHERE</span> LOC_ID <span class="operator">=</span> <span class="number">10</span> <span class="keyword">OR</span> LOC_ID <span class="operator">=</span> <span class="number">20</span> <span class="keyword">OR</span> LOC_ID <span class="operator">=</span> <span class="number">30</span></span><br><span class="line">高效</span><br><span class="line"><span class="keyword">SELECT</span>… <span class="keyword">FROM</span> LOCATION <span class="keyword">WHERE</span> LOC_IN <span class="keyword">IN</span> (<span class="number">10</span>,<span class="number">20</span>,<span class="number">30</span>);</span><br></pre></td></tr></table></figure>
+<ol start="25" type="1">
+<li><p>避免在索引列上使用IS NULL和IS NOT NULL</p>
+<p>避免在索引中使用任何可以为空的列，ORACLE将无法使用该索引．对于单列索引，如果列包含空值，索引中将不存在此记录.
+对于复合索引，如果每个列都为空，索引中同样不存在此记录.
+如果至少有一个列不为空，则记录存在于索引中．举例:
+如果唯一性索引建立在表的A列和B列上,
+并且表中存在一条记录的A,B值为(123,null) ,
+ORACLE将不接受下一条具有相同A,B值（123,null）的记录(插入).
+然而如果所有的索引列都为空，ORACLE将认为整个键值为空而空不等于空.
+因此你可以插入1000 条具有相同键值的记录,当然它们都是空!
+因为空值不存在于索引列中,所以WHERE子句中对索引列进行空值比较将使ORACLE停用该索引.</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效: (索引失效)</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPARTMENT <span class="keyword">WHERE</span> DEPT_CODE <span class="keyword">IS</span> <span class="keyword">NOT</span> <span class="keyword">NULL</span>;</span><br><span class="line">高效: (索引有效)</span><br><span class="line"><span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> DEPARTMENT <span class="keyword">WHERE</span> DEPT_CODE <span class="operator">&gt;=</span><span class="number">0</span>;</span><br></pre></td></tr></table></figure>
+<ol start="26" type="1">
+<li><p>总是使用索引的第一个列</p>
+<p>如果索引是建立在多个列上, 只有在它的第一个列(leading
+column)被where子句引用时,优化器才会选择使用该索引.
+这也是一条简单而重要的规则，当仅引用索引的第二个列时,优化器使用了全表扫描而忽略了索引。</p></li>
+<li><p>用UNION-ALL 替换UNION ( 如果有可能的话)</p>
+<p>当SQL
+语句需要UNION两个查询结果集合时,这两个结果集合会以UNION-ALL的方式被合并,
+然后在输出最终结果前进行排序. 如果用UNION ALL替代UNION,
+这样排序就不是必要了. 效率就会因此得到提高. 需要注意的是，UNION ALL
+将重复输出两个结果集合中相同记录. 因此各位还是要从业务需求分析使用UNION
+ALL的可行性. UNION
+将对结果集合排序,这个操作会使用到SORT_AREA_SIZE这块内存.
+对于这块内存的优化也是相当重要的.
+下面的SQL可以用来查询排序的消耗量</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效：</span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line"><span class="keyword">UNION</span></span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br><span class="line"><span class="keyword">UNION</span> <span class="keyword">ALL</span></span><br><span class="line"><span class="keyword">SELECT</span> ACCT_NUM, BALANCE_AMT</span><br><span class="line"><span class="keyword">FROM</span> DEBIT_TRANSACTIONS</span><br><span class="line"><span class="keyword">WHERE</span> TRAN_DATE <span class="operator">=</span> <span class="string">&#x27;31-DEC-95&#x27;</span></span><br></pre></td></tr></table></figure>
+<ol start="28" type="1">
+<li><p>用WHERE替代ORDER BY</p>
+<p>ORDER BY 子句只在两种严格的条件下使用索引.</p>
+<p>ORDER BY中所有的列必须包含在相同的索引中并保持在索引中的排列顺序.</p>
+<p>ORDER BY中所有的列必须定义为非空.</p>
+<p>WHERE子句使用的索引和ORDER BY子句中所使用的索引不能并列.</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效: (索引不被使用)</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_CODE <span class="keyword">FROM</span> DEPT <span class="keyword">ORDER</span> <span class="keyword">BY</span> DEPT_TYPE</span><br><span class="line">高效: (使用索引)</span><br><span class="line"><span class="keyword">SELECT</span> DEPT_CODE <span class="keyword">FROM</span> DEPT <span class="keyword">WHERE</span> DEPT_TYPE <span class="operator">&gt;</span> <span class="number">0</span></span><br></pre></td></tr></table></figure>
+<ol start="29" type="1">
+<li><p>避免改变索引列的类型</p>
+<p>当比较不同数据类型的数据时,
+ORACLE自动对列进行简单的类型转换.</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"># 假设 EMPNO是一个数值类型的索引列.</span><br><span class="line"># 低效</span><br><span class="line"><span class="keyword">SELECT</span> … <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> EMPNO <span class="operator">=</span> <span class="string">&#x27;123&#x27;</span></span><br><span class="line"># 高效</span><br><span class="line"><span class="keyword">SELECT</span> … <span class="keyword">FROM</span> EMP <span class="keyword">WHERE</span> EMPNO <span class="operator">=</span> <span class="number">123</span></span><br></pre></td></tr></table></figure>
+<ol start="30" type="1">
+<li><p>需要当心的WHERE子句</p>
+<p>某些SELECT 语句中的WHERE子句不使用索引. 这里有一些例子.</p>
+<ol type="1">
+<li><code>!=</code> 将不使用索引. 记住, 索引只能告诉你什么存在于表中,
+而不能告诉你什么不存在于表中.</li>
+<li><code>¦ ¦</code>是字符连接函数. 就象其他函数那样, 停用了索引.</li>
+<li><code>+</code>是数学函数. 就象其他数学函数那样, 停用了索引.</li>
+<li>相同的索引列不能互相比较,这将会启用全表扫描.</li>
+</ol></li>
+<li><ol type="a">
+<li><p>如果检索数据量超过30%的表中记录数.使用索引将没有显著的效率提高.</p></li>
+<li><p>在特定情况下, 使用索引也许会比全表扫描慢,
+但这是同一个数量级上的区别.
+而通常情况下,使用索引比全表扫描要块几倍乃至几千倍!</p></li>
+</ol></li>
+<li><p>避免使用耗费资源的操作</p>
+<p>带有DISTINCT,UNION,MINUS,INTERSECT,ORDER
+BY的SQL语句会启动SQL引擎执行耗费资源的排序(SORT)功能.
+DISTINCT需要一次排序操作, 而其他的至少需要执行两次排序. 通常, 带有UNION,
+MINUS , INTERSECT的SQL语句都可以用其他方式重写.
+如果你的数据库的SORT_AREA_SIZE调配得好, 使用UNION , MINUS,
+INTERSECT也是可以考虑的, 毕竟它们的可读性很强。</p></li>
+<li><p>优化GROUP BY</p>
+<p>提高GROUP BY 语句的效率, 可以通过将不需要的记录在GROUP BY
+之前过滤掉.下面两个查询返回相同结果但第二个明显就快了许多.</p></li>
+</ol>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">低效:</span><br><span class="line"><span class="keyword">SELECT</span> JOB, <span class="built_in">AVG</span>(SAL)</span><br><span class="line"><span class="keyword">FROM</span> EMP</span><br><span class="line"><span class="keyword">GROUP</span> <span class="keyword">by</span> JOB</span><br><span class="line"><span class="keyword">HAVING</span> JOB <span class="operator">=</span> <span class="string">&#x27;PRESIDENT&#x27;</span></span><br><span class="line"><span class="keyword">OR</span> JOB <span class="operator">=</span> <span class="string">&#x27;MANAGER&#x27;</span></span><br><span class="line">高效:</span><br><span class="line"><span class="keyword">SELECT</span> JOB, <span class="built_in">AVG</span>(SAL)</span><br><span class="line"><span class="keyword">FROM</span> EMP</span><br><span class="line"><span class="keyword">WHERE</span> JOB <span class="operator">=</span> <span class="string">&#x27;PRESIDENT&#x27;</span></span><br><span class="line"><span class="keyword">OR</span> JOB <span class="operator">=</span> <span class="string">&#x27;MANAGER&#x27;</span></span><br><span class="line"><span class="keyword">GROUP</span> <span class="keyword">by</span> JOB</span><br></pre></td></tr></table></figure>
+<pre><code>​</code></pre>
 ]]></content>
       <categories>
-        <category>Environment</category>
+        <category>Database</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>Tensorflow</tag>
-        <tag>Caffe</tag>
-        <tag>风格迁移</tag>
+        <tag>数据库</tag>
+        <tag>SQL</tag>
       </tags>
   </entry>
   <entry>
@@ -7049,79 +6728,257 @@ listening port be bound for local use only, while an empty address or
 <p>ssh部分参数介绍</p>
 <p>可以用在上面的命令中</p>
 <ul>
-<li><code>-f</code>
-后台认证用户密码，即不用登录到远程主机。通常与-N连用</li>
-<li><code>-C</code> 压缩数据传输</li>
-<li><code>-N</code> 不执行脚本或者命令。通常与-f连用</li>
-<li><code>-g</code> 允许远程主机连接到建立的转发端口。</li>
-<li><code>-q</code>
-静默模式，使大多数警告和诊断消息被压制。（如果有警告信息，不进行输出）</li>
+<li><code>-f</code>
+后台认证用户密码，即不用登录到远程主机。通常与-N连用</li>
+<li><code>-C</code> 压缩数据传输</li>
+<li><code>-N</code> 不执行脚本或者命令。通常与-f连用</li>
+<li><code>-g</code> 允许远程主机连接到建立的转发端口。</li>
+<li><code>-q</code>
+静默模式，使大多数警告和诊断消息被压制。（如果有警告信息，不进行输出）</li>
+</ul>
+<p>举个栗子</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 本地端口转发。命令执行后，不登录到远程服务器，压缩数据传输，也不输出警告信息</span></span><br><span class="line">ssh -gfCNL 2222:localhost:22 user@remotehost -q</span><br><span class="line"></span><br><span class="line"><span class="comment"># 如果将命令放到了后台，关掉进程需要使用kill命令</span></span><br></pre></td></tr></table></figure>
+<p>如果网络不稳定可以考虑使用
+<code>autossh</code>。这个需要额外安装，使用的时候将
+<code>ssh</code>替换为
+<code>autossh</code>即可。它的工作原理简单来说，就是有个超时机制，如果中断，便重新连接。</p>
+<h2 id="用途介绍">用途介绍</h2>
+<ol type="1">
+<li>本地端口转发。假设有两台服务器A和B，B需要访问A上面的一个应用（网站）。</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在B上使用本地端口转发，将某个端口转发给A应用的端口。</span></span><br><span class="line"><span class="comment"># B机器</span></span><br><span class="line">ssh -L 2333:IP_A:80 localuser@localhost</span><br><span class="line"><span class="comment"># 此时访问localhost:2333等于pi_A:80</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 比如A为公网IP（11.11.11.11），在6000端口开启某项服务</span></span><br><span class="line"><span class="comment"># B通过在本机使用, 本地用户名user</span></span><br><span class="line">ssh -L 2333:11.11.11.11:6000 user@localhost</span><br><span class="line"><span class="comment"># 即可通过localhost:2333来代替11.11.11.11</span></span><br></pre></td></tr></table></figure>
+<ol start="2" type="1">
+<li><p>本地端口转发高级版（多机版）。同上面的假设，但是现在有三台或者三台以上的服务器A、B、C……这个时候，使用
+<code>-g</code>参数，运行远程主机连接转发端口。这样便可以达到C访问B再访问A的目的。注意：确保ABC三台服务器网络连接是安全的，请谨慎。</p></li>
+<li><p>远程端口转发。假设有一台服务器（公网）A，两台内网计算机B、C。</p></li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># B、C均可连接A，但是其他两两之间不能相互连接。</span></span><br><span class="line"><span class="comment"># 现在需要实现两两之间都能互相连接</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># A连接到B、C</span></span><br><span class="line"><span class="comment"># B、C远程端口转发，A机器（11.11.11.11），用户名user</span></span><br><span class="line"><span class="comment"># B机器</span></span><br><span class="line">ssh -R 2221:localhost:22 user@11.11.11.11</span><br><span class="line"><span class="comment"># C机器</span></span><br><span class="line">ssh -R 2222:localhost:22 user@11.11.11.11</span><br><span class="line"></span><br><span class="line"><span class="comment"># 这样在A机器执行如下命令就能连接B、C了</span></span><br><span class="line"><span class="comment"># 连接B</span></span><br><span class="line">ssh userB@localhost -p2221</span><br><span class="line"><span class="comment"># 连接C</span></span><br><span class="line">ssh userC@localhost -p2222</span><br><span class="line"></span><br><span class="line"><span class="comment"># B、C之间互连</span></span><br><span class="line"><span class="comment"># 其实就是通过A作为跳板来登录，即先登录A</span></span><br><span class="line">ssh user@11.11.11.11</span><br><span class="line"><span class="comment"># 再登录B or C即可</span></span><br></pre></td></tr></table></figure>
+<ol start="4" type="1">
+<li>动态端口转发。在本地使用这个命令连接到服务器之后，便可以使用服务器的SOCK5代理来上网。需要在浏览器或者系统上进行下面的设置。</li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A机器（user，11.11.11.11）</span></span><br><span class="line"><span class="comment"># B机器上执行</span></span><br><span class="line">ssh -D 1234 user@11.11.11.11</span><br><span class="line"><span class="comment"># 通过设置SOCKS5代理，socks5://127.0.0.1:1234，就可以通过A机器的流量上网。可查询ip来检验是否成功</span></span><br></pre></td></tr></table></figure>
+<p>注意事项：</p>
+<ul>
+<li>端口转发是通过ssh连接建立的，所以关闭了端口，端口转发也会关闭</li>
+<li>选择远程端口号的时候，一般是无权绑定
+<code>1-1023</code>端口的，只能使用管理员权限才能绑定。一般是使用
+<code>1024-65535</code>之间的一个端口</li>
+</ul>
+<h2 id="内网流量转发">内网流量转发</h2>
+<p>内网转发内网流量，A，C都在局域网下，二者不可相互连接。B是公网服务器，A，C皆可连接B，但B不可以连接A，C。现在需求是A借助C的流量上网。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A机器执行。user和remotehost是B机器的用户名和ip</span></span><br><span class="line">ssh -L localhost:3000:localhost:2000 user@remotehost</span><br><span class="line"><span class="comment"># C机器执行。user和remotehost是B机器的用户名和ip</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># B机器执行。user是C机器的用户名</span></span><br><span class="line">ssh -D 2000 -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"><span class="comment"># 或两条命令，其实就是两条并一条</span></span><br><span class="line"><span class="comment"># C机器</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># A机器</span></span><br><span class="line">ssh -L localhost:3000:localhost:2000 user@remotehost -t ssh -D 2000 -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"><span class="comment"># A机器上代理：SOCKS5://127.0.0.1:3000</span></span><br></pre></td></tr></table></figure>
+<h2 id="一键命令">一键命令</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A在内网，B在公网，已通过B连接A。-R命令</span></span><br><span class="line"><span class="comment"># BUSER和BHOST对应正常连接B的SSH用户名和密码，AUSER为B连接A的用户名，port为-R参数的port</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 此时需要在C机器上，先连接B，再通过B连接A。C机器执行命令。</span></span><br><span class="line">ssh -t BUSER@BHOST ssh AUSER@localhost -p port</span><br><span class="line"></span><br><span class="line"><span class="comment"># or （Win10上测试失败）</span></span><br><span class="line">ssh BUSER@BHOST -J AUSER@localhost:port</span><br></pre></td></tr></table></figure>
+<figure class="highlight powershell"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 一键连接内网服务器，windows下的config配置</span></span><br><span class="line"><span class="comment"># jump为跳板服务器，jump_server_ip、username、port分别对应跳板机的ip、用户名和连接端口</span></span><br><span class="line"><span class="comment"># target_server为内网服务器，基本同上，ProxyCommand配置如下所示，需要使用ssh命令的绝对路径</span></span><br><span class="line">Host jump</span><br><span class="line">    ForwardAgent yes</span><br><span class="line">    HostName jump_server_ip</span><br><span class="line">    User username</span><br><span class="line">    Port port</span><br><span class="line"></span><br><span class="line">Host target_server</span><br><span class="line">    ForwardAgent yes</span><br><span class="line">    HostName target_server_ip</span><br><span class="line">    User username</span><br><span class="line">    Port port</span><br><span class="line">    ProxyCommand C:\\Windows\\System32\\OpenSSH\\ssh.exe <span class="literal">-q</span> <span class="literal">-W</span> %<span class="built_in">h</span>:%p jump</span><br></pre></td></tr></table></figure>
+<h2 id="注意事项">注意事项</h2>
+<p>win10开启ssh服务器端，1. 应用和功能：下载openssh服务端；2.
+服务：开启openssh server服务；3.
+连接用户名win10系统盘的用户文件下的文件名，比如我的是Administrator（即和公用文件夹并列的文件夹）。</p>
+<p>参考文章</p>
+<p><a
+href="http://www.ruanyifeng.com/blog/2011/12/ssh_port_forwarding.html">SSH原理与运用（二）：远程操作与端口转发</a></p>
+<p><a
+href="https://www.ibm.com/developerworks/cn/linux/l-cn-sshforward/">实战
+SSH 端口转发</a></p>
+<p><a
+href="http://blog.creke.net/722.html">SSH隧道与端口转发及内网穿透</a></p>
+<p><a
+href="https://blog.mythsman.com/2017/01/14/1/">利用反向ssh从外网访问内网主机</a></p>
+<p><a
+href="http://www.pchou.info/linux/2015/11/01/ssh-tunnel.html">SSH隧道翻墙的原理和实现</a></p>
+<p><a
+href="https://blog.fundebug.com/2017/04/24/ssh-port-forwarding/">玩转SSH端口转发</a></p>
+<p><a
+href="https://www.ibm.com/developerworks/community/blogs/5144904d-5d75-45ed-9d2b-cf1754ee936a/entry/20160911?lang=en">openssh的三种tcp端口转发参数</a></p>
+<p><a
+href="https://github.com/microsoft/vscode-remote-release/issues/230">https://github.com/microsoft/vscode-remote-release/issues/230</a></p>
+]]></content>
+      <categories>
+        <category>Software</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>端口转发</tag>
+        <tag>ssh</tag>
+        <tag>内网穿透</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Ubuntu20.04安装深度学习环境及配置个人环境小记</title>
+    <url>/hexo_blog/2021/10/30/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Ubuntu20.04%E5%AE%89%E8%A3%85%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0%E7%8E%AF%E5%A2%83%E5%8F%8A%E9%85%8D%E7%BD%AE%E4%B8%AA%E4%BA%BA%E7%8E%AF%E5%A2%83%E5%B0%8F%E8%AE%B0/</url>
+    <content><![CDATA[<p>Ubuntu20.04安装深度学习环境及配置个人环境小记</p>
+<span id="more"></span>
+<h3 id="目标配置环境">目标配置环境</h3>
+<p>系统为Ubuntu 20.04,
+显卡为3090，结合其他服务器的环境，准备配置如下环境</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">nvidia-driver-460 <span class="comment"># 英伟达驱动，最低推荐版本为这个</span></span><br><span class="line">CUDA 11.2 (gcc 7) <span class="comment"># 30系显卡应该换为cuda11版本以上，20系显卡可以用cuda10.2版本</span></span><br><span class="line">cudnn 8.6.1</span><br><span class="line"></span><br><span class="line">pytorch 1.10</span><br><span class="line">torchvision 0.11.1</span><br><span class="line">tensorflow 2.5.0</span><br></pre></td></tr></table></figure>
+<h3 id="准备工作">准备工作</h3>
+<p>刚装的系统，啥也没有，所以按照个人习惯先安装一些软件。为方便起见，后文非特殊说明，均用root用户完成安装。可以直接复制下文至sh文件中，更改权限以直接运行。（请保证软件源可正常连接）</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 第一步: 安装一些必备的软件</span></span><br><span class="line">add-apt-repository ppa:graphics-drivers/ppa</span><br><span class="line">apt update -y &amp;&amp; apt upgrade -y</span><br><span class="line">apt install -y git autossh python3-pip vim tmux gpustat tree ranger build-essential gcc-7 g++-7 make curl htop ipython3 zhs</span><br><span class="line"><span class="comment"># gcc与g++降级</span></span><br><span class="line">update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100</span><br><span class="line">update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第二步：安装nvidia驱动</span></span><br><span class="line"><span class="comment"># 禁用nouveau</span></span><br><span class="line"><span class="built_in">chmod</span> 666 /etc/modprobe.d/blacklist.conf</span><br><span class="line"><span class="comment"># 增加相应配置, 若已追加则记得注释下面这一行命令</span></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&quot;</span></span><br><span class="line"><span class="string">blacklist vga16fb</span></span><br><span class="line"><span class="string">blacklist nouveau</span></span><br><span class="line"><span class="string">blacklist rivafb</span></span><br><span class="line"><span class="string">blacklist rivatv</span></span><br><span class="line"><span class="string">blacklist nvidiafb</span></span><br><span class="line"><span class="string">&quot;</span></span><br><span class="line">&gt;&gt; /etc/modprobe.d/blacklist.conf</span><br><span class="line"><span class="comment"># 更新配置</span></span><br><span class="line">update-initramfs -u</span><br><span class="line">apt install nvidia-driver-460 -y <span class="comment"># 英伟达460驱动，按需更改</span></span><br></pre></td></tr></table></figure>
+<p>执行完成之后，重启服务器。使用<code>nvidia-smi</code>查看gpu是否正常，或使用安装好的<code>gpustat</code></p>
+<h3 id="cuda及cudnn安装">cuda及cudnn安装</h3>
+<h4 id="下载">下载</h4>
+<p>这里需要从官网下载安装包，cuda安装包可以直接通过命令下载。cudnn需要注册英伟达开发者账号，下载四个文件</p>
+<ul>
+<li><p><code>cudnn-11.2-linux-x64-v8.1.0.77</code></p></li>
+<li><p><code>libcudnn8_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
+<li><p><code>libcudnn8-dev_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
+<li><p><code>libcudnn8-samples_8.1.0.77-1+cuda11.2_amd64.deb</code></p></li>
+</ul>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget https://developer.download.nvidia.com/compute/cuda/11.2.2/local_installers/cuda_11.2.2_460.32.03_linux.run &amp;&amp; <span class="built_in">chmod</span> 755 cuda_11.2.2_460.32.03_linux.run</span><br><span class="line"></span><br><span class="line">./cuda_11.2.2_460.32.03_linux.run</span><br></pre></td></tr></table></figure>
+<p>下载完直接运行，安装过程中需要把Drivers这项取消，因为自带了显卡驱动。</p>
+<h4 id="环境变量">环境变量</h4>
+<p>在/etc/bash.bashrc或/etc/profile前加入如下两行。如果profile不行就换bashrc。一般是编辑<code>vim ~/.bashrc</code>下，但为保证所有用户都生效，所以需要在<code>/etc</code>目录下编辑</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">export</span> PATH=/usr/local/cuda-11.2/bin<span class="variable">$&#123;PATH:+:<span class="variable">$&#123;PATH&#125;</span>&#125;</span></span><br><span class="line"><span class="built_in">export</span> LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64<span class="variable">$&#123;LD_LIBRARY_PATH:+:<span class="variable">$&#123;LD_LIBRARY_PATH&#125;</span>&#125;</span></span><br></pre></td></tr></table></figure>
+<p>至此，cuda安装完成</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 检测是否安装成功</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/cuda/samples/1_Utilities/deviceQuery</span><br><span class="line">make clean &amp;&amp; make -j4 &amp;&amp; ./deviceQuery</span><br><span class="line"><span class="comment"># 最后一行输出，Result = PASS表示成功</span></span><br></pre></td></tr></table></figure>
+<h4 id="cudnn配置">cudnn配置</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tar -zxvf cudnn-11.2-linux-x64-v8.1.0.77.tgz </span><br><span class="line"><span class="built_in">cp</span> cuda/include/cudnn.h /usr/local/cuda/include </span><br><span class="line"><span class="built_in">cp</span> cuda/lib64/libcudnn* /usr/local/cuda/lib64  </span><br><span class="line"><span class="built_in">chmod</span> a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn*  </span><br><span class="line"></span><br><span class="line">dpkg -i libcudnn8_8.1.0.77-1+cuda11.2_amd64.deb      </span><br><span class="line">dpkg -i libcudnn8-dev_8.1.0.77-1+cuda11.2_amd64.deb  </span><br><span class="line">dpkg -i libcudnn8-samples_8.1.0.77-1+cuda11.2_amd64.deb </span><br><span class="line"></span><br><span class="line"><span class="comment"># cudnn检测，更高版本的cudnn需要把cudnn.h换成cudnn_version.h</span></span><br><span class="line"><span class="built_in">cat</span> /usr/local/cuda/include/cudnn.h | grep CUDNN_MAJOR -A 2</span><br><span class="line"><span class="comment"># 利用程序检查，可能需要重启</span></span><br><span class="line"><span class="built_in">cd</span> /usr/src/cudnn_samples_v7/conv_sample</span><br><span class="line">make clean &amp;&amp; make -j4 &amp;&amp; ./conv_sample</span><br><span class="line"><span class="comment"># 最后一行输出，Test passed！表示成功</span></span><br></pre></td></tr></table></figure>
+<p>通过<code>ldconfig</code>可检查动态链接库是否链接正确。若有输出，可以使用<code>ln -sf</code>重新链接。</p>
+<h3 id="python库安装">Python库安装</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">pip3 install tensorflow-gpu==2.5.0 -i https://pypi.tuna.tsinghua.edu.cn/simple</span><br><span class="line">pip3 install torch==1.10.0+cu113 torchvision==0.11.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html</span><br><span class="line">pip3 install black tqdm flask sklearn</span><br></pre></td></tr></table></figure>
+<h4 id="检查是否安装成功">检查是否安装成功</h4>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.cuda.is_available()) </span><br><span class="line"><span class="built_in">print</span>(torch.backends.cudnn.is_available()) </span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(torch.__version__)</span><br><span class="line"><span class="built_in">print</span>(torch.version.cuda)</span><br><span class="line"><span class="built_in">print</span>(torch.backends.cudnn.version())</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> tensorflow <span class="keyword">as</span> tf</span><br><span class="line">gpu_device_name = tf.test.gpu_device_name()</span><br><span class="line"><span class="built_in">print</span>(gpu_device_name)   </span><br><span class="line">tf.test.is_gpu_available() <span class="comment"># 输出True</span></span><br></pre></td></tr></table></figure>
+<p>若tf未检测到GPU，可以看输出过程中的哪些动态库文件加载失败，手动配置一下。</p>
+<p>至此，深度学习安装环境配置完成。</p>
+<h3 id="个人环境">个人环境</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建用户</span></span><br><span class="line">adduser xxx</span><br><span class="line"><span class="comment"># 输入配置的密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 增加root权限</span></span><br><span class="line"><span class="built_in">chmod</span> u+w /etc/sudoers</span><br><span class="line">vim /etc/sudoers</span><br><span class="line">​```</span><br><span class="line">root    ALL=(ALL:ALL) ALL</span><br><span class="line"><span class="comment"># 增加一行</span></span><br><span class="line">xxx    ALL=(ALL:ALL) ALL</span><br><span class="line">​```</span><br><span class="line"><span class="built_in">chmod</span> u-w /etc/sudoers</span><br></pre></td></tr></table></figure>
+<h4 id="ssh文件配置">ssh文件配置</h4>
+<p>方便起见直接把其他服务器的<code>.ssh</code>文件夹下的<code>id_rsa</code>，<code>id_rsa.pub</code>和<code>authorized_keys</code>复制过来。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 复制完成之后，修改对应权限才能使用</span></span><br><span class="line"><span class="built_in">chmod</span> 755 ~/.ssh/  </span><br><span class="line"><span class="built_in">chmod</span> 600 ~/.ssh/id_rsa ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys</span><br><span class="line"><span class="built_in">chmod</span> 644 ~/.ssh/known_hosts</span><br></pre></td></tr></table></figure>
+<h4 id="oh-my-zsh">oh-my-zsh</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载并安装</span></span><br><span class="line">wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | sh</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装插件</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/zsh-users/zsh-autosuggestions <span class="variable">$&#123;ZSH_CUSTOM:-~/.oh-my-zsh/custom&#125;</span>/plugins/zsh-autosuggestions</span><br><span class="line">git <span class="built_in">clone</span> https://github.com/zsh-users/zsh-syntax-highlighting.git <span class="variable">$&#123;ZSH_CUSTOM:-~/.oh-my-zsh/custom&#125;</span>/plugins/zsh-syntax-highlighting</span><br><span class="line"></span><br><span class="line"><span class="comment"># vim ~/.zshrc, 最后一行加入如下配置</span></span><br><span class="line">plugins=(</span><br><span class="line">        git</span><br><span class="line">        zsh-autosuggestions</span><br><span class="line">        zsh-syntax-highlighting</span><br><span class="line">)</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/custom/plugins/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/custom/plugins/zsh-autosuggestions/zsh-autosuggestions.zsh</span><br><span class="line"><span class="built_in">source</span> <span class="variable">$ZSH</span>/oh-my-zsh.sh</span><br></pre></td></tr></table></figure>
+<h4 id="配置vim">配置vim</h4>
+<p>编辑<code>.vimrc</code></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">set</span> number</span><br><span class="line"></span><br><span class="line">syntax on</span><br><span class="line">colorscheme darkblue</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">syntax <span class="built_in">enable</span></span><br><span class="line">filetype plugin on</span><br><span class="line"><span class="built_in">let</span> g:go_disable_autoinstall = 0</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_functions = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_methods = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_structs = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_operators = 1</span><br><span class="line"><span class="built_in">let</span> g:go_highlight_build_constraints = 1</span><br><span class="line"><span class="string">&quot; &quot;</span> syntax on</span><br><span class="line"><span class="string">&quot;&quot;</span> filetype plugin indent on</span><br><span class="line"><span class="built_in">let</span> g:pymode_python = <span class="string">&#x27;python3&#x27;</span></span><br><span class="line"><span class="built_in">set</span> number</span><br><span class="line"><span class="built_in">set</span> cin</span><br><span class="line"><span class="built_in">set</span> sw=8</span><br><span class="line"><span class="built_in">set</span> ts=8</span><br><span class="line"><span class="built_in">set</span> sm</span><br><span class="line">colorscheme darkblue</span><br><span class="line"><span class="built_in">set</span> viminfo=<span class="string">&#x27;10,\&quot;100,:20,%,n~/.viminfo </span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">au BufReadPost * if line(&quot;&#x27;</span>\&quot;<span class="string">&quot;) &gt; 0|if line(&quot;</span><span class="string">&#x27;\&quot;&quot;) &lt;= line(&quot;$&quot;)|exe(&quot;norm &#x27;</span>\&quot;<span class="string">&quot;)|else|exe &quot;</span>norm $<span class="string">&quot;|endif|endif</span></span><br></pre></td></tr></table></figure>
+<h4 id="远程连接">远程连接</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">autossh -gfCNR 远程端口:localhost:22（默认本地ssh端口） 公网用户名@公网IP</span><br></pre></td></tr></table></figure>
+<p>需要提前配好免密登录，远程端口在公网服务器上需要开放防火墙。</p>
+<p>远程连接登录<code>ssh 服务器用户@公网IP -p 远程端口</code></p>
+]]></content>
+      <categories>
+        <category>Linux</category>
+      </categories>
+      <tags>
+        <tag>CUDA</tag>
+        <tag>PyTorch</tag>
+        <tag>Ubuntu</tag>
+        <tag>Tensorflow</tag>
+        <tag>CUDNN</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>在Hadoop基础上Hive的安装</title>
+    <url>/hexo_blog/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Hive在Hadoop上的安装及一些基本的Hive操作</p>
+<span id="more"></span>
+<p>写在开头 环境 Hadoop单机 Centos7 Hadoop-2.7.3
+hadoop位置：/usr/loacl/hadoop</p>
+<p>参考文章 <a
+href="http://yanliu.org/2015/08/13/Hadoop%E9%9B%86%E7%BE%A4%E4%B9%8BHive%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">Hadoop集群之Hive安装配置</a></p>
+<h2 id="下载hive">下载Hive</h2>
+<p>下载源码包 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#在hadoop目录下操作</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop</span><br><span class="line"></span><br><span class="line"><span class="comment">#用wget下载</span></span><br><span class="line">wget http://mirrors.cnnic.cn/apache/hive/hive-1.2.1/apache-hive-1.2.1-bin.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#也可以在图形界面下下载之后，上传压缩包</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#解压缩包</span></span><br><span class="line">tar -zxvf apache-hive-1.2.1-bin.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#配置环境变量</span></span><br><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#位置在之前配置的变量之后，大概12行左右，因为之前配置了jdk和hadoop变量</span></span><br><span class="line"><span class="built_in">export</span> HIVE_HOME=/usr/local/hadoop/apache-hive-1.2.2-bin</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$HIVE_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"><span class="comment">#适当位置添加&quot;$HIVE_HOME/bin:&quot;</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#使文件生效</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure></p>
+<h2 id="安装mysql">安装MySQL</h2>
+<p>MariaDB数据库管理系统是MySQL的一个分支，主要由开源社区在维护，采用GPL授权许可。开发这个分支的原因之一是：甲骨文公司收购了MySQL后，有将MySQL闭源的潜在风险，因此社区采用分支的方式来避开这个风险。MariaDB的目的是完全兼容MySQL，包括API和命令行，使之能轻松成为MySQL的代替品。</p>
+<p>在Centos7下，使用命令安装mysql，会安装成mariadb。</p>
+<p>在这里先介绍安装MariaDB，原因如上。</p>
+<h3 id="安装mariadb">安装MariaDB</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#安装</span></span><br><span class="line">yum install mariadb-server mariadb</span><br><span class="line"></span><br><span class="line"><span class="comment">#启动</span></span><br><span class="line">systemctl start mariadb</span><br><span class="line"></span><br><span class="line"><span class="comment">#进入MySQL</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#命令行变成如下，可能有点不习惯</span></span><br><span class="line">MariaDB [(none)]&gt;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#P.S.</span></span><br><span class="line"><span class="comment">#相关命令</span></span><br><span class="line">systemctl start mariadb <span class="comment">#启动MariaDB</span></span><br><span class="line">systemctl stop mariadb <span class="comment">#停止MariaDB</span></span><br><span class="line">systemctl restart mariadb <span class="comment">#重启MariaDB</span></span><br><span class="line">systemctl <span class="built_in">enable</span> mariadb <span class="comment">#设置开机启动</span></span><br></pre></td></tr></table></figure>
+<h3 id="正式安装mysql">正式安装MySQL</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#下载，在这里使用的是命令行下载，也建议在图形界面下载，然后上传至服务器</span></span><br><span class="line">wget http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm</span><br><span class="line"></span><br><span class="line"><span class="comment">#使用rpm安装</span></span><br><span class="line">rpm -ivh mysql-community-release-el7-5.noarch.rpm</span><br><span class="line"></span><br><span class="line"><span class="comment">#使用yum安装mysql-community-server</span></span><br><span class="line">yum install mysql-community-server</span><br><span class="line"></span><br><span class="line"><span class="comment">#启动服务</span></span><br><span class="line">service mysqld start</span><br><span class="line"></span><br><span class="line"><span class="comment">#进入mysql，第一次进入无密码</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#命令行变成如下</span></span><br><span class="line">mysql&gt;</span><br></pre></td></tr></table></figure>
+<h3 id="配置mysql">配置MySQL</h3>
+<p>编码配置 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入配置文件，若未安装vim，建议先使用命令yum install vim安装vim</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"></span><br><span class="line"><span class="comment">#最后加上编码配置</span></span><br><span class="line">[mysql]</span><br><span class="line">default-character-set =utf8</span><br><span class="line"></span><br><span class="line"><span class="comment">#此处字符编码必须和/usr/share/mysql/charsets/Index.xml中一致。</span></span><br><span class="line"><span class="comment">#不过一般情况下使用的都是utf8</span></span><br></pre></td></tr></table></figure></p>
+<p>设置密码 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#下面三种方法需要进入mysql</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#方法一</span></span><br><span class="line">mysql&gt;insert into user(host,user,password) values(<span class="string">&#x27;%&#x27;</span>,<span class="string">&#x27;user_name&#x27;</span>,password(<span class="string">&quot;password&quot;</span>);</span><br><span class="line"><span class="comment">#方法二</span></span><br><span class="line">mysql&gt;<span class="built_in">set</span> password <span class="keyword">for</span> user_name = password(<span class="string">&quot;password&quot;</span>);</span><br><span class="line"><span class="comment">#方法三</span></span><br><span class="line">mysql&gt;grant all on *.* to user_name@% identified by <span class="string">&quot;password&quot;</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment">#下面这一种方法可直接在shell下设置密码</span></span><br><span class="line">mysqladmin -u root password <span class="string">&quot;password&quot;</span></span><br></pre></td></tr></table></figure> 远程连接</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入mysql</span></span><br><span class="line">mysql -u root -p</span><br><span class="line"></span><br><span class="line"><span class="comment">#把在所有数据库的所有表的所有权限赋值给位于所有IP地址的root用户。</span></span><br><span class="line">mysql&gt; grant all privileges on *.* to root@<span class="string">&#x27;%&#x27;</span>identified by <span class="string">&quot;password&quot;</span>;</span><br></pre></td></tr></table></figure>
+<p>P.S.上文引号中的user_name表示数据库的用户名，password表示对应用户的密码。即这两项是由读者自行定义的。</p>
+<h3 id="常见问题及解决方案">常见问题及解决方案</h3>
+<h4 id="mysql创建用户失败">Mysql创建用户失败</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># mysql在my.ini的配置文件中设置了严格模式，所以我们需要进行修改</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 第一步，寻找配置文件</span></span><br><span class="line">whereis my.ini</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第二步，根据上一步结果进行vim编辑</span></span><br><span class="line">vim /../my.ini</span><br><span class="line"></span><br><span class="line"><span class="comment"># 第三步，在vim里面搜索sql-mode,删除STRICT_TRANS_TABLES，保存退出即可</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 第四步，保险起见，使刚刚的配置文件立即生效</span></span><br><span class="line"><span class="built_in">source</span> /../my.ini</span><br></pre></td></tr></table></figure>
+<h4 id="支持中文">支持中文</h4>
+<ul>
+<li><p>Centos7</p>
+<p><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 编辑文件</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"><span class="comment"># 在对应[xx]下增加修改如下代码</span></span><br><span class="line">[client]</span><br><span class="line">port = 3306</span><br><span class="line">socket = /var/lib/mysql/mysql.sock</span><br><span class="line">default-character-set=utf8</span><br><span class="line"></span><br><span class="line">[mysqld]</span><br><span class="line">port = 3306</span><br><span class="line">socket = /var/lib/mysql/mysql.sock</span><br><span class="line">default-storage-engine=INNODB</span><br><span class="line">character-set-server=utf8</span><br><span class="line">collation-server=utf8_general_ci</span><br><span class="line"></span><br><span class="line">[mysql]</span><br><span class="line">no-auto-rehash</span><br><span class="line">default-character-set=utf8</span><br><span class="line"><span class="comment"># 保存退出，重启服务</span></span><br><span class="line"><span class="comment"># 重新登陆mysql检查是否成功，方法见下</span></span><br></pre></td></tr></table></figure></p></li>
+<li><p>Ubuntu16.04</p>
+<p><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">sudo</span> vim /etc/mysql/mysql.conf.d/mysqld.cnf</span><br><span class="line"><span class="comment"># 在对应[xx]下增加以下内容，如果不存在[xx]自行增加</span></span><br><span class="line">[mysqld]</span><br><span class="line">character_set_server=utf8</span><br><span class="line">[mysql]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[mysql.server]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[mysqld_safe]</span><br><span class="line">default-character-set=utf8</span><br><span class="line">[client]</span><br><span class="line">default-character-set=utf8</span><br><span class="line"><span class="comment"># 重启mysql</span></span><br><span class="line">service mysql restart</span><br><span class="line"><span class="comment"># 进入mysql查看配置参数</span></span><br><span class="line">mysql -uroot -p</span><br><span class="line"><span class="comment"># 查看database 的value变为utf8即可</span></span><br><span class="line">&gt;show variables like <span class="string">&#x27;%character%&#x27;</span>;</span><br><span class="line"></span><br><span class="line"><span class="comment"># p.s.如果之前创建了表，表编码不会改变</span></span><br></pre></td></tr></table></figure></p></li>
 </ul>
-<p>举个栗子</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 本地端口转发。命令执行后，不登录到远程服务器，压缩数据传输，也不输出警告信息</span></span><br><span class="line">ssh -gfCNL 2222:localhost:22 user@remotehost -q</span><br><span class="line"></span><br><span class="line"><span class="comment"># 如果将命令放到了后台，关掉进程需要使用kill命令</span></span><br></pre></td></tr></table></figure>
-<p>如果网络不稳定可以考虑使用
-<code>autossh</code>。这个需要额外安装，使用的时候将
-<code>ssh</code>替换为
-<code>autossh</code>即可。它的工作原理简单来说，就是有个超时机制，如果中断，便重新连接。</p>
-<h2 id="用途介绍">用途介绍</h2>
+<h4
+id="mysql报错1366---incorrect-integer-value-xxxxxxx-at-row-1">MySQL报错“1366
+- Incorrect integer value: '' XXXXXXX' at row 1 ”</h4>
+<p>修改方法:（两种,建议第二种）</p>
 <ol type="1">
-<li>本地端口转发。假设有两台服务器A和B，B需要访问A上面的一个应用（网站）。</li>
+<li><p>命令行。set names gbk；(此为设置通信编码)</p></li>
+<li><p><code>my.cnf</code> 中查找sql-mode</p></li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在B上使用本地端口转发，将某个端口转发给A应用的端口。</span></span><br><span class="line"><span class="comment"># B机器</span></span><br><span class="line">ssh -L 2333:IP_A:80 localuser@localhost</span><br><span class="line"><span class="comment"># 此时访问localhost:2333等于pi_A:80</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 比如A为公网IP（11.11.11.11），在6000端口开启某项服务</span></span><br><span class="line"><span class="comment"># B通过在本机使用, 本地用户名user</span></span><br><span class="line">ssh -L 2333:11.11.11.11:6000 user@localhost</span><br><span class="line"><span class="comment"># 即可通过localhost:2333来代替11.11.11.11</span></span><br></pre></td></tr></table></figure>
-<ol start="2" type="1">
-<li><p>本地端口转发高级版（多机版）。同上面的假设，但是现在有三台或者三台以上的服务器A、B、C……这个时候，使用
-<code>-g</code>参数，运行远程主机连接转发端口。这样便可以达到C访问B再访问A的目的。注意：确保ABC三台服务器网络连接是安全的，请谨慎。</p></li>
-<li><p>远程端口转发。假设有一台服务器（公网）A，两台内网计算机B、C。</p></li>
+<p>将 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">sql-mode=<span class="string">&quot;STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION&quot;</span>，</span><br></pre></td></tr></table></figure> 修改为 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">sql-mode=<span class="string">&quot;NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION&quot;</span>，</span><br></pre></td></tr></table></figure> 重启mysql后即可</p>
+<h4 id="关于my.cnf和my.ini的说明">关于my.cnf和my.ini的说明</h4>
+<p>my.cnf常见于Linux系统，my.ini常见与windows系统，二者都是属于mysql的配置文件。一般好像在一个系统下就是只出现一种配置文件，具体区别没有深入了解，修改配置文件，根据自己的系统进行查找修改配置文件即可</p>
+<h4 id="mysql交互环境自动补全">Mysql交互环境自动补全</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 修改配置文件</span></span><br><span class="line">vim /etc/my.cnf</span><br><span class="line"><span class="comment"># 在[mysql]部分添加auto-rehash</span></span><br><span class="line"><span class="comment"># 保存退出，重启</span></span><br><span class="line">service mysqld restart</span><br></pre></td></tr></table></figure>
+<h2 id="配置hive">配置Hive</h2>
+<p>我们之前在hadoop目录下安装了Hive，位置为<code>/usr/local/hadoop/apache-hive-1.2.1-bin</code></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入hive配置目录下</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop/apache-hive-1.2.1-bin/conf</span><br><span class="line"></span><br><span class="line"><span class="comment">#修改hive-default.xml.template</span></span><br><span class="line"><span class="comment">#首先复制</span></span><br><span class="line"><span class="built_in">cp</span> hive-default.xml.template hive-default.xml</span><br><span class="line"><span class="comment">#修改文件</span></span><br><span class="line">vim hive-default.xml</span><br><span class="line"></span><br><span class="line"><span class="comment"># 1. 第一步将&lt;configuration&gt;&lt;/configuration&gt;中内容删除大概是18-3908行</span></span><br><span class="line"><span class="comment"># vim删除命令--&gt;:18,3908d</span></span><br><span class="line"><span class="comment"># 2.将下面&lt;configuration&gt;&lt;/configuration&gt;中内容复制进去,分别将3306喝9083前的user_name改为当前的用户名</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionURL&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;jdbc:mysql://user_name:3306/hive?createDatabaseIfNotExist=<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;JDBC connect string <span class="keyword">for</span> a JDBC metastore&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionDriverName&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;com.mysql.jdbc.Driver&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;Driver class name <span class="keyword">for</span> a JDBC metastore&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line"></span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionUserName&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hive&lt;value&gt;</span><br><span class="line">        &lt;description&gt;username to use against metastore database&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;javax.jdo.option.ConnectionPassword&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hive&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;password to use against metastore database&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line"></span><br><span class="line">    &lt;property&gt;</span><br><span class="line">    &lt;name&gt;hive.metastore.uris&lt;/name&gt;</span><br><span class="line">    &lt;value&gt;thrift://user_name:9083&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br></pre></td></tr></table></figure>
+<h2 id="下载jdbc">下载JDBC</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#命令行下载,也可以使用图形界面上传文件</span></span><br><span class="line"> wget http://cdn.mysql.com/Downloads/Connector-J/mysql-connector-java-5.1.36.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment">#将文件复制进Hive的lib目录下，原因，Hive的自带的那个版本低，可能失效</span></span><br><span class="line"> <span class="built_in">cp</span> mysql-connector-java-5.1.33-bin.jar /usr/local/hadoop/apache-hive-1.2.1-bin/lib/</span><br></pre></td></tr></table></figure>
+<p>启动Hive <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">hive --service metastore &amp;</span><br><span class="line">jps</span><br><span class="line"><span class="comment">#结果会多出一个进程</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#进入hive目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop/apache-hive-1.2.1-bin/bin</span><br><span class="line"><span class="comment">#启动hive，可能有点慢</span></span><br><span class="line">hive</span><br><span class="line"></span><br><span class="line"><span class="comment">#若出现hive的命令行即代表成功，如下</span></span><br><span class="line">hive&gt;</span><br></pre></td></tr></table></figure></p>
+<h2 id="常见问题及解决方案-1">常见问题及解决方案</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">Logging initialized using configuration <span class="keyword">in</span> jar:file:/home/hadoop/apache-hive-1.2.1-bin/lib/hive-common-1.2.1.jar!/hive-log4j.properties</span><br><span class="line">Exception <span class="keyword">in</span> thread <span class="string">&quot;main&quot;</span> java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient</span><br><span class="line">        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:522)</span><br></pre></td></tr></table></figure>
+<p>解决方法，直接关闭防火墙</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#这里的系统为Centos7，所以使用此命令</span></span><br><span class="line">systemctl stop firewalld</span><br></pre></td></tr></table></figure>
+<p>hive metastore 启动出错解决 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查看与hive相关进程是否启动</span></span><br><span class="line">ps -ef | grep hive</span><br><span class="line"><span class="comment"># kill相关进程，为进程号</span></span><br><span class="line"><span class="built_in">kill</span> num</span><br><span class="line"><span class="comment"># 重新启动</span></span><br><span class="line">./hive</span><br></pre></td></tr></table></figure></p>
+<p>更多问题见此文章 <a
+href="http://blog.csdn.net/freedomboy319/article/details/44828337">Hive常见问题汇总</a></p>
+<h2 id="hive的学习笔记">Hive的学习笔记</h2>
+<h3 id="新建表">新建表</h3>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 新建一张表，名为“test”，里面有“name”、“id”两类，分别是“string”、“int”的数据类型，以“|”隔开一列，表是作为textfile的。</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">CREATE</span> <span class="keyword">TABLE</span> test (name String, id <span class="type">int</span>) <span class="type">ROW</span> FORMAT DELIMITED FIELDS TERMINATED <span class="keyword">BY</span> <span class="string">&#x27;|&#x27;</span> STORED <span class="keyword">AS</span> TEXTFILE;</span><br></pre></td></tr></table></figure>
+<h3 id="加载表">加载表</h3>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 从本地的/home/user/test.txt文件，将数据加载进test这个表</span></span><br><span class="line">hive<span class="operator">&gt;</span> LOAD DATA <span class="keyword">LOCAL</span> INPATH <span class="string">&#x27;/home/user/test.txt&#x27;</span> OVERWRITE <span class="keyword">INTO</span> <span class="keyword">TABLE</span> test;</span><br></pre></td></tr></table></figure>
+<h3 id="关联表">关联表</h3>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 将两张表通过一个或多个字段关联在一起</span></span><br><span class="line"><span class="comment">-- 将test_a和test_b表的id字段关联在一起，再加上“name”字段，组成一个新表</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> test_a.id, test_b.name <span class="keyword">FROM</span> test_a, test_b <span class="keyword">JOIN</span> test_b <span class="keyword">ON</span> (test_a.id <span class="operator">=</span> test_b.id);</span><br></pre></td></tr></table></figure>
+<h3 id="保存表">保存表</h3>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line"><span class="comment">-- 由于hive下执行任务之后，并不会保存数据，所以我们使用INSERT命令来保存命令</span></span><br><span class="line"><span class="comment">-- 插入数据到test表中</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">INSERT</span> OVERWRITE <span class="keyword">TABLE</span> test</span><br><span class="line">     <span class="operator">&gt;</span> ...</span><br><span class="line"></span><br><span class="line"><span class="comment">-- 保存表到hdfs上,保存输出结果到hdfs下的/out/目录下</span></span><br><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">INSERT</span> OVERWRITE <span class="keyword">LOCAL</span> DIRECTORY &quot;/out/&quot;</span><br><span class="line">    <span class="operator">&gt;</span> ...</span><br></pre></td></tr></table></figure>
+<h3 id="排序问题">排序问题</h3>
+<ol type="1">
+<li>order by</li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># B、C均可连接A，但是其他两两之间不能相互连接。</span></span><br><span class="line"><span class="comment"># 现在需要实现两两之间都能互相连接</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># A连接到B、C</span></span><br><span class="line"><span class="comment"># B、C远程端口转发，A机器（11.11.11.11），用户名user</span></span><br><span class="line"><span class="comment"># B机器</span></span><br><span class="line">ssh -R 2221:localhost:22 user@11.11.11.11</span><br><span class="line"><span class="comment"># C机器</span></span><br><span class="line">ssh -R 2222:localhost:22 user@11.11.11.11</span><br><span class="line"></span><br><span class="line"><span class="comment"># 这样在A机器执行如下命令就能连接B、C了</span></span><br><span class="line"><span class="comment"># 连接B</span></span><br><span class="line">ssh userB@localhost -p2221</span><br><span class="line"><span class="comment"># 连接C</span></span><br><span class="line">ssh userC@localhost -p2222</span><br><span class="line"></span><br><span class="line"><span class="comment"># B、C之间互连</span></span><br><span class="line"><span class="comment"># 其实就是通过A作为跳板来登录，即先登录A</span></span><br><span class="line">ssh user@11.11.11.11</span><br><span class="line"><span class="comment"># 再登录B or C即可</span></span><br></pre></td></tr></table></figure>
-<ol start="4" type="1">
-<li>动态端口转发。在本地使用这个命令连接到服务器之后，便可以使用服务器的SOCK5代理来上网。需要在浏览器或者系统上进行下面的设置。</li>
+<figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test <span class="keyword">ORDER</span> <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure>
+<ol start="2" type="1">
+<li><p>sort by <figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test SORT <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure></p></li>
+<li><p>distribute by <figure class="highlight sql"><table><tr><td class="code"><pre><span class="line">hive<span class="operator">&gt;</span> <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> test <span class="keyword">ORDER</span> <span class="keyword">BY</span> name DISTRIBUTE <span class="keyword">BY</span> id;</span><br></pre></td></tr></table></figure></p></li>
+<li><p>DISTRIBUTE BY with SORT BY DISTRIBUTE BY 和 GROUP BY
+有点类似，DISTRIBUTE BY控制reduce如何处理数据，而SORT
+BY控制reduce中的数据如何排序。 注意：hive要求DISTRIBUTE BY语句出现在SORT
+BY语句之前。</p></li>
+<li><p>Cluster By</p></li>
 </ol>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A机器（user，11.11.11.11）</span></span><br><span class="line"><span class="comment"># B机器上执行</span></span><br><span class="line">ssh -D 1234 user@11.11.11.11</span><br><span class="line"><span class="comment"># 通过设置SOCKS5代理，socks5://127.0.0.1:1234，就可以通过A机器的流量上网。可查询ip来检验是否成功</span></span><br></pre></td></tr></table></figure>
-<p>注意事项：</p>
+<p>cluster by 除了具有 distribute by 的功能外还兼具 sort by 的功能。
+默认升序排序，但DISTRIBUTE BY的字段和SORT
+BY的字段必须相同，且不能指定排序规则。</p>
+<p>总结：</p>
 <ul>
-<li>端口转发是通过ssh连接建立的，所以关闭了端口，端口转发也会关闭</li>
-<li>选择远程端口号的时候，一般是无权绑定
-<code>1-1023</code>端口的，只能使用管理员权限才能绑定。一般是使用
-<code>1024-65535</code>之间的一个端口</li>
+<li><code>ORDER BY</code>是全局排序，但在数据量大的情况下，花费时间会很长</li>
+<li><code>SORT BY</code>是将<code>reduce</code>的单个输出进行排序，不能保证全局有序</li>
+<li><code>DISTRIBUTE BY</code>
+可以按指定字段将数据划分到不同的<code>reduce</code>中</li>
+<li>当<code>DISTRIBUTE BY</code> 的字段和
+<code>SORT BY</code>的字段相同时，可以用<code>CLUSTER BY</code>来代替<code>DISTRIBUTE BY with SORT BY</code>。</li>
 </ul>
-<h2 id="内网流量转发">内网流量转发</h2>
-<p>内网转发内网流量，A，C都在局域网下，二者不可相互连接。B是公网服务器，A，C皆可连接B，但B不可以连接A，C。现在需求是A借助C的流量上网。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A机器执行。user和remotehost是B机器的用户名和ip</span></span><br><span class="line">ssh -L localhost:3000:localhost:2000 user@remotehost</span><br><span class="line"><span class="comment"># C机器执行。user和remotehost是B机器的用户名和ip</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># B机器执行。user是C机器的用户名</span></span><br><span class="line">ssh -D 2000 -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"><span class="comment"># 或两条命令，其实就是两条并一条</span></span><br><span class="line"><span class="comment"># C机器</span></span><br><span class="line">ssh -R 2222:localhost:22 user@remotehost</span><br><span class="line"><span class="comment"># A机器</span></span><br><span class="line">ssh -L localhost:3000:localhost:2000 user@remotehost -t ssh -D 2000 -p 2222 user@localhost</span><br><span class="line"></span><br><span class="line"><span class="comment"># A机器上代理：SOCKS5://127.0.0.1:3000</span></span><br></pre></td></tr></table></figure>
-<h2 id="一键命令">一键命令</h2>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># A在内网，B在公网，已通过B连接A。-R命令</span></span><br><span class="line"><span class="comment"># BUSER和BHOST对应正常连接B的SSH用户名和密码，AUSER为B连接A的用户名，port为-R参数的port</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 此时需要在C机器上，先连接B，再通过B连接A。C机器执行命令。</span></span><br><span class="line">ssh -t BUSER@BHOST ssh AUSER@localhost -p port</span><br><span class="line"></span><br><span class="line"><span class="comment"># or （Win10上测试失败）</span></span><br><span class="line">ssh BUSER@BHOST -J AUSER@localhost:port</span><br></pre></td></tr></table></figure>
-<figure class="highlight powershell"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 一键连接内网服务器，windows下的config配置</span></span><br><span class="line"><span class="comment"># jump为跳板服务器，jump_server_ip、username、port分别对应跳板机的ip、用户名和连接端口</span></span><br><span class="line"><span class="comment"># target_server为内网服务器，基本同上，ProxyCommand配置如下所示，需要使用ssh命令的绝对路径</span></span><br><span class="line">Host jump</span><br><span class="line">    ForwardAgent yes</span><br><span class="line">    HostName jump_server_ip</span><br><span class="line">    User username</span><br><span class="line">    Port port</span><br><span class="line"></span><br><span class="line">Host target_server</span><br><span class="line">    ForwardAgent yes</span><br><span class="line">    HostName target_server_ip</span><br><span class="line">    User username</span><br><span class="line">    Port port</span><br><span class="line">    ProxyCommand C:\\Windows\\System32\\OpenSSH\\ssh.exe <span class="literal">-q</span> <span class="literal">-W</span> %<span class="built_in">h</span>:%p jump</span><br></pre></td></tr></table></figure>
-<h2 id="注意事项">注意事项</h2>
-<p>win10开启ssh服务器端，1. 应用和功能：下载openssh服务端；2.
-服务：开启openssh server服务；3.
-连接用户名win10系统盘的用户文件下的文件名，比如我的是Administrator（即和公用文件夹并列的文件夹）。</p>
-<p>参考文章</p>
-<p><a
-href="http://www.ruanyifeng.com/blog/2011/12/ssh_port_forwarding.html">SSH原理与运用（二）：远程操作与端口转发</a></p>
-<p><a
-href="https://www.ibm.com/developerworks/cn/linux/l-cn-sshforward/">实战
-SSH 端口转发</a></p>
-<p><a
-href="http://blog.creke.net/722.html">SSH隧道与端口转发及内网穿透</a></p>
-<p><a
-href="https://blog.mythsman.com/2017/01/14/1/">利用反向ssh从外网访问内网主机</a></p>
-<p><a
-href="http://www.pchou.info/linux/2015/11/01/ssh-tunnel.html">SSH隧道翻墙的原理和实现</a></p>
-<p><a
-href="https://blog.fundebug.com/2017/04/24/ssh-port-forwarding/">玩转SSH端口转发</a></p>
-<p><a
-href="https://www.ibm.com/developerworks/community/blogs/5144904d-5d75-45ed-9d2b-cf1754ee936a/entry/20160911?lang=en">openssh的三种tcp端口转发参数</a></p>
-<p><a
-href="https://github.com/microsoft/vscode-remote-release/issues/230">https://github.com/microsoft/vscode-remote-release/issues/230</a></p>
+<h3 id="hive常用函数">hive常用函数</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 以下无特殊说明，返回值皆为string</span></span><br><span class="line"><span class="comment"># 字符串连接</span></span><br><span class="line">hive&gt;concat(strA, strB, ...)</span><br><span class="line"><span class="comment"># 带分隔符的字符串连接，即连接之后字符串存在分隔符，strSEP表示分隔符</span></span><br><span class="line">hive&gt;concat_wd(strSEP, strA, strB, ...)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串截取</span></span><br><span class="line"><span class="comment"># 从整型数num0开始截取，截取到结尾为止</span></span><br><span class="line">hive&gt;substr(strA, num0)</span><br><span class="line"><span class="comment"># 从整型数num0开始截取，截取到num1为止</span></span><br><span class="line">hive&gt;substr(strA, num0, num1)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 字符串转大写</span></span><br><span class="line">hive&gt;upper(str)</span><br><span class="line"><span class="comment"># 字符串转小写</span></span><br><span class="line">hive&gt;lower(str)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 除去字符串中空格</span></span><br><span class="line">hive&gt;trim(str)</span><br><span class="line"><span class="comment"># 除去字符串左边空格</span></span><br><span class="line">hive&gt;ltrim(str)</span><br><span class="line"><span class="comment"># 除去字符串右边空格</span></span><br><span class="line">hive&gt;rtrim(str)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则替换字符</span></span><br><span class="line"><span class="comment"># 将strA中符合strRE的部分替换为strB</span></span><br><span class="line">hive&gt;regexp_replace(strA, strRE, strB)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 重复字符串</span></span><br><span class="line"><span class="comment"># str为字符串，num为重复次数</span></span><br><span class="line">hive&gt;repeat(str, num)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 分割字符串</span></span><br><span class="line"><span class="comment"># 以strpat为分割符，返回字符串数组</span></span><br><span class="line">hive&gt;spilt(str, strpat)</span><br></pre></td></tr></table></figure>
+<h3 id="hive导出数据">hive导出数据</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导出至本地，content表示本地目录</span></span><br><span class="line">hive&gt;insert overwrite <span class="built_in">local</span> directory <span class="string">&#x27;content&#x27;</span></span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 导出至hdfs，content表示hdfs目录</span></span><br><span class="line">hive&gt;insert overwrite directory <span class="string">&#x27;content&#x27;</span></span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 导出至hive另一张表othertable，前提hive中存在这张表</span></span><br><span class="line">hive&gt;insert overwrite othertable</span><br><span class="line">    &gt;<span class="keyword">select</span> * from table_name;</span><br></pre></td></tr></table></figure>
+<h3 id="转换类型">转换类型</h3>
+<p>cast(xxx as xx) 将xxx类型转换为xx类型</p>
+<p>下表为是否可转换类型的说明 <img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/hadoop/20170612195815151.png" /></p>
+<p>注：由于表格比较大，这里对一些比较长的字符串进行缩写，ts是timestamp的缩写,bl是boolean的缩写,sl是smallint的缩写,dm是decimal的缩写,vc是varchar的缩写,ba是binary的缩写。</p>
 ]]></content>
       <categories>
-        <category>Software</category>
+        <category>Environment</category>
       </categories>
       <tags>
-        <tag>Linux</tag>
-        <tag>端口转发</tag>
-        <tag>ssh</tag>
-        <tag>内网穿透</tag>
+        <tag>Hadoop</tag>
+        <tag>Hive</tag>
+        <tag>大数据</tag>
       </tags>
   </entry>
   <entry>
@@ -7140,1284 +6997,1550 @@ href="https://github.com/microsoft/vscode-remote-release/issues/230">https://git
 <h2 id="github-hook配置">Github Hook配置</h2>
 <p>Github配置位置，项目的<code>Settings-&gt;Webhooks-&gt;Add webhook</code>。填写对应的信息</p>
 <ul>
-<li>Payload URL: 域名+端口（flask服务配置）</li>
-<li>Content type：json</li>
-<li>Secret：自定义密钥，在脚本中配置</li>
-<li><strong>Which events would you like to trigger this
-webhook?</strong>：仅在推送时进行操作</li>
+<li>Payload URL: 域名+端口（flask服务配置）</li>
+<li>Content type：json</li>
+<li>Secret：自定义密钥，在脚本中配置</li>
+<li><strong>Which events would you like to trigger this
+webhook?</strong>：仅在推送时进行操作</li>
+</ul>
+<p>服务器上的python脚本</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding=utf-8</span></span><br><span class="line"><span class="keyword">from</span> flask <span class="keyword">import</span> Flask, request</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"><span class="keyword">from</span> pprint <span class="keyword">import</span> pprint</span><br><span class="line"><span class="keyword">import</span> hmac</span><br><span class="line">app = Flask(__name__)</span><br><span class="line"></span><br><span class="line">repo_path = <span class="string">&#x27;&#x27;</span> <span class="comment"># git仓库地址</span></span><br><span class="line">secret = <span class="string">&#x27;&#x27;</span>	   <span class="comment"># github配置的密钥</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">encryption</span>(<span class="params">data</span>):</span><br><span class="line">    key = secret.encode(<span class="string">&#x27;utf-8&#x27;</span>)</span><br><span class="line">    obj = hmac.new(key, msg=data, digestmod=<span class="string">&#x27;sha1&#x27;</span>)</span><br><span class="line">    <span class="keyword">return</span> obj.hexdigest()</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@app.route(<span class="params"><span class="string">&quot;/&quot;</span>, methods=[<span class="string">&quot;GET&quot;</span>, <span class="string">&quot;POST&quot;</span>]</span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">hello</span>():</span><br><span class="line">    post_data = request.data</span><br><span class="line">    token = encryption(post_data)</span><br><span class="line">    <span class="comment"># 认证签名是否有效</span></span><br><span class="line">    signature = request.headers.get(<span class="string">&#x27;X-Hub-Signature&#x27;</span>, <span class="string">&#x27;&#x27;</span>).split(<span class="string">&#x27;=&#x27;</span>)[-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">if</span> signature != token:</span><br><span class="line">        <span class="keyword">return</span> <span class="string">&#x27;failed&#x27;</span></span><br><span class="line"></span><br><span class="line">    os.system(<span class="string">&#x27;cd &#123;&#125; &amp;&amp; git pull&#x27;</span>.<span class="built_in">format</span>(repo_path))</span><br><span class="line">    <span class="keyword">return</span> <span class="string">&#x27;success&#x27;</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    app.run(host=<span class="string">&quot;0.0.0.0&quot;</span>, port=port) <span class="comment"># 开启的port，记得打开对应端口的防火墙</span></span><br></pre></td></tr></table></figure>
+<p>若推送失败，可在服务器手动<code>git pull</code>，或者在<code>webhook</code>地址，<code>[Recent Deliveries]</code>中进行<code>Redeliver</code></p>
+]]></content>
+      <categories>
+        <category>Note</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>Nginx</tag>
+        <tag>Github</tag>
+        <tag>Webhook</tag>
+        <tag>HTTPS</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>Hadoop伪分布式安装（Centos7）</title>
+    <url>/hexo_blog/2017/04/09/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/Hadoop%E4%BC%AA%E5%88%86%E5%B8%83%E5%BC%8F%E5%AE%89%E8%A3%85%EF%BC%88Centos7%EF%BC%89/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了Hadoop的伪分布式安装和一些基础操作</p>
+<span id="more"></span>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/hadoop/wKiom1SFSNnAeaEUAADD3ZcjTjw828.jpg" /></p>
+<h2 id="文章开头">文章开头</h2>
+<p>环境配置：Centos7</p>
+<p>Hadoop版本：hadoop：2.7.3</p>
+<p>JDK版本：jdk-8u111-linux-x64.tar.gz</p>
+<h2 id="安装之前">安装之前</h2>
+<ol type="1">
+<li><p>下载hadoop安装包, <a
+href="http://hadoop.apache.org/releases.html">下载网址</a>。进入官网之后，点击对应版本的“binary”，之后点击链接下载即可。</p></li>
+<li><p>jdk。通过官网下载jdk</p></li>
+<li><p>通过Xshell，上传两个安装包至/usr/local/。服务器与本地下载和上传需要lrzsz这个软件</p></li>
+</ol>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">yum install lrzsz</span><br></pre></td></tr></table></figure>
+<p>之后可手动拖动文件至服务器当前目录下，也可通过命令
+“rz”来打开窗口，从而进行上传。 （下载命令“sz”）</p>
+<h2 id="解压缩及配置">解压缩及配置</h2>
+<p>在<code>/usr/local/</code>目录下进行解压 <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#请根据下载版本，进行解压缩</span></span><br><span class="line">tar -zxvf jdk-8u111-linux-x64.tar.gz</span><br><span class="line">tar -zxvf hadoop-2.7.3.tar.gz</span><br></pre></td></tr></table></figure></p>
+<h3
+id="配置java环境变量可与配置hadoop环境变量配置一起进行">配置Java环境变量（可与配置Hadoop环境变量配置一起进行）</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#第十行左右输入命令，注意版本</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/local/jdk1.8.0_111</span><br><span class="line"><span class="built_in">export</span> PATH=.:<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$PATH</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>
+<h3 id="配置hadoop环境变量">配置Hadoop环境变量</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vim /etc/profile</span><br><span class="line"><span class="comment">#在刚刚JAVA_HOME之下一行，注意版本</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#此处的PATH承接上面Java环境变量的配置，即是在原来基础上进行增加而不是另起一行</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_HOME=/usr/local/hadoop-2.7.3</span><br><span class="line"><span class="built_in">export</span> PATH=.:<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$HADOOP_HOME</span>/bin:<span class="variable">$HADOOP_HOME</span>/sbin:<span class="variable">$PATH</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#更新设置</span></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>
+<h3 id="配置启动hadoop">配置启动Hadoop</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#进入配置文件目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop-2.7.3/etc/hadoop/</span><br><span class="line"><span class="comment">#修改配置文件</span></span><br><span class="line">vim hadoop-env.sh</span><br><span class="line"><span class="comment">#第二十五行</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/usr/local/jdk1.8.0_111</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim core-site.xml</span><br><span class="line"><span class="comment">#添加&lt;configuration&gt;&lt;/configuration&gt;中间的配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 指定HDFS老大（namenode）的通信地址 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">	    &lt;name&gt;fs.defaultFS&lt;/name&gt;</span><br><span class="line">	    &lt;value&gt;hdfs://hadoopbonc1:9000&lt;/value&gt;</span><br><span class="line">    &lt;!-- 指定hadoop运行时产生文件的存储路径 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hadoop.tmp.dir&lt;/name&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#此处fs.defaultFS的vaule写本机主机名，即第一个value的hadoopbonc1换成主机名</span></span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim hdfs-site.xml</span><br><span class="line"><span class="comment">#添加如上配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 设置hdfs副本数量 --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">         &lt;name&gt;dfs.replication&lt;/name&gt;</span><br><span class="line">         &lt;value&gt;1&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br></pre></td></tr></table></figure>
+<h3 id="配置ssh免密码登陆密码互通">配置SSH免密码登陆（密码互通）</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#配置前</span></span><br><span class="line">ssh localhost</span><br><span class="line"><span class="comment">#回车后，要求输入本机登陆密码</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#配置主机名</span></span><br><span class="line">vim /etc/sysconfig/network</span><br><span class="line"><span class="built_in">source</span> /etc/sysconfig/network</span><br><span class="line"></span><br><span class="line"><span class="comment">#配置hosts</span></span><br><span class="line">vim /etc/hosts</span><br><span class="line">ssh-keygen</span><br><span class="line"></span><br><span class="line"><span class="comment">#把公钥和私钥复制到相应的节点（因为本机是单节点，所以复制到本机）</span></span><br><span class="line"><span class="comment">#此处的hadoopbonc1换成主机名</span></span><br><span class="line">ssh-copy-id hadoopbonc1</span><br></pre></td></tr></table></figure>
+<h3 id="hdfs启动与停止">hdfs启动与停止</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#第一次启动需要格式化，理由跟磁盘第一次使用需要格式化一样</span></span><br><span class="line"><span class="comment">#以后启动就不需要格式化</span></span><br><span class="line">hdfs namenode -format</span><br><span class="line"><span class="comment">#启动hdfs</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#在hadoop目录下</span></span><br><span class="line"><span class="built_in">cd</span> /usr/locate/hadoop-2.7.3/</span><br><span class="line"><span class="comment">#启动</span></span><br><span class="line">./sbin/start-dfs.sh</span><br><span class="line"><span class="comment">#输入jps检查是否成功</span></span><br><span class="line">jps</span><br><span class="line"><span class="comment">#出现四行顺序不定，分别为SecondaryNameNode、DataNode、NameNode、Jps，即表示成功</span></span><br><span class="line"><span class="comment">#打开浏览器输入：服务器ip:50070</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#若无响应则需要开放50070端口，则需开放50070端口</span></span><br><span class="line">firewall-cmd --zone=public --add-port=50070/tcp --permanent</span><br><span class="line">firewall-cmd --reload</span><br><span class="line"></span><br><span class="line"><span class="comment">#hdfs的停止</span></span><br><span class="line"><span class="comment">#同样在hadoop目录下</span></span><br><span class="line">./sbin/stop-dfs.sh</span><br><span class="line"></span><br></pre></td></tr></table></figure>
+<h3 id="配置和启动yarn">配置和启动YARN</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#切换配置文件目录</span></span><br><span class="line"><span class="built_in">cd</span> /usr/local/hadoop-2.7.3/etc/hadoop/</span><br><span class="line"><span class="built_in">mv</span> mapred-site.xml.template mapred-site.xml</span><br><span class="line">vim mapred-site.xml</span><br><span class="line"><span class="comment">#添加&lt;configuration&gt;&lt;/configuration&gt;中间的配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;!-- 通知框架MR使用YARN --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;mapreduce.framework.name&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;yarn&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line">vim yarn-site.xml</span><br><span class="line"><span class="comment">#添加如上配置</span></span><br><span class="line">&lt;configuration&gt;</span><br><span class="line">	&lt;!-- reducer取数据的方式是mapreduce_shuffle --&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">         &lt;name&gt;yarn.nodemanager.aux-services&lt;/name&gt;</span><br><span class="line">         &lt;value&gt;mapreduce_shuffle&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br><span class="line"><span class="comment">#保存退出</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#启动YARN</span></span><br><span class="line">start-yarn.sh</span><br><span class="line"><span class="comment">#输入jps检查是否成功</span></span><br><span class="line">jps</span><br><span class="line"><span class="comment">#出现六行顺序不定，分别为SecondaryNameNode、DataNode、NameNode、Jps、ResourceManager、NodeManager，即表示成功</span></span><br><span class="line"><span class="comment">#打开浏览器输入：服务器ip:8088</span></span><br><span class="line"><span class="comment">#若无响应，可参照上文中的开放50070端口，开放8088端口，只需将50070换为8088即可</span></span><br></pre></td></tr></table></figure>
+<h2 id="测试">测试</h2>
+<p>在本地新建一个文件，如在<code>/home/user/</code>下新建<code>words.txt</code>，内容如下
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">hello world</span><br><span class="line">hello hadoop</span><br><span class="line">hello csdn</span><br><span class="line">hello</span><br></pre></td></tr></table></figure></p>
+<p>正式进行测试。命令如下： <figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#在hdfs根目录下新建test目录</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">mkdir</span> /test</span><br><span class="line"></span><br><span class="line"><span class="comment">#查看hdfs根目录下的目录结构</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">ls</span> /</span><br><span class="line"></span><br><span class="line"><span class="comment">#将本地文件上传至/test/目录下</span></span><br><span class="line">bin/hdfs dfs -put /home/user/words.txt /test/</span><br><span class="line"></span><br><span class="line"><span class="comment">#运行wordcount</span></span><br><span class="line">bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar wordcount /test/words.txt /test/out</span><br><span class="line"></span><br><span class="line"><span class="comment">#在/test/目录下生成了一个名为out的文件目录，查看一下/out/目录下的文件</span></span><br><span class="line">bin/hdfs dfs -<span class="built_in">ls</span> /test/out</span><br><span class="line"></span><br><span class="line"><span class="comment">#结果保存在part-r-00000，查看一下运行结果</span></span><br><span class="line">bin/hdfs fs -<span class="built_in">cat</span> /test/out/part-r-00000</span><br></pre></td></tr></table></figure></p>
+<h2 id="hdfs的常用操作命令">HDFS的常用操作命令</h2>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment">#常用操作：</span></span><br><span class="line"><span class="comment">#HDFS shell</span></span><br><span class="line"><span class="comment">#查看帮助</span></span><br><span class="line">hadoop fs -<span class="built_in">help</span> &lt;cmd&gt;</span><br><span class="line"><span class="comment">#上传</span></span><br><span class="line">hadoop fs -<span class="built_in">cat</span> &lt;hdfs上的路径&gt;</span><br><span class="line"><span class="comment">#查看文件列表</span></span><br><span class="line">hadoop fs -<span class="built_in">ls</span> /</span><br><span class="line"><span class="comment">#下载文件</span></span><br><span class="line">hadoop fs -get &lt;hdfs上的路径&gt; &lt;linux上文件&gt;</span><br></pre></td></tr></table></figure>
+<p>下一篇：<a href="/hexo_blog/2017/05/07/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E5%9C%A8Hadoop%E5%9F%BA%E7%A1%80%E4%B8%8AHive%E7%9A%84%E5%AE%89%E8%A3%85/" title="在Hadoop基础上Hive的安装">在Hadoop基础上Hive的安装</a></p>
+]]></content>
+      <categories>
+        <category>Environment</category>
+      </categories>
+      <tags>
+        <tag>Linux</tag>
+        <tag>Hadoop</tag>
+        <tag>大数据</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>风格迁移转换的两种实现</title>
+    <url>/hexo_blog/2017/08/14/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85/%E9%A3%8E%E6%A0%BC%E8%BF%81%E7%A7%BB%E8%BD%AC%E6%8D%A2%E7%9A%84%E4%B8%A4%E7%A7%8D%E5%AE%9E%E7%8E%B0/</url>
+    <content><![CDATA[<p>本文介绍了如何在Ubuntu16.04环境下搭建caffe和tensorflow，并利用caffe和tensorflow实现了风格转换。同时，介绍了如何训练新的风格模型。</p>
+<span id="more"></span>
+<p><strong>style-transfer</strong></p>
+<p>这个主要是利用caffe+model+code直接实现任意风格的转换。</p>
+<p>最后实现需借助</p>
+<ul>
+<li>一张风格图片</li>
+<li>待转换风格的目标图片</li>
+<li>训练模型</li>
+</ul>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">python style.py -s &lt;style_image&gt; -c &lt;content_image&gt; -m &lt;model_name&gt; -g 0</span><br></pre></td></tr></table></figure>
+<p>优势：</p>
+<ul>
+<li>可以实现任意风格的转换</li>
+<li>可拓展性强 缺点：</li>
+<li>耗时长（真的很长，如果用CPU的话）</li>
+</ul>
+<p><strong>fast-neural-style-tensorflow</strong></p>
+<p>风格快速迁移转换</p>
+<p>顾名思义，这个比前者能够更快的进行风格转换，速度因机器而异，不过效率比前者确确实实提高了很多倍</p>
+<p>最后实现需借助</p>
+<ul>
+<li>一个训练好的风格模型</li>
+<li>待转换的风格图片</li>
+</ul>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding</span></span><br><span class="line"> python eval.py --model_file &lt;path of ckpt-done&gt; --image_file &lt;path of image&gt;</span><br></pre></td></tr></table></figure>
+<p>优势：</p>
+<ul>
+<li>速度相对来说很快</li>
+<li>环境搭建相对来说更容易 缺点：</li>
+<li>可拓展性更弱</li>
+<li>只能转换固定几种风格</li>
+<li>训练新风格模型时间长（比前者生成时间更长）</li>
+</ul>
+<h2 id="style-transfer的实现">style-transfer的实现</h2>
+<p><strong>环境介绍</strong></p>
+<p>Ubuntu16.04 + CPU + python2.7 + caffe</p>
+<h3 id="搭建caffe环境">搭建caffe环境</h3>
+<h4 id="介绍">介绍</h4>
+<p>Caffe是一个清晰而高效的深度学习框架。Caffe是纯粹的C++/CUDA架构，支持命令行、Python和MATLAB接口；可以在CPU和GPU直接无缝切换。</p>
+<p>优势：</p>
+<ul>
+<li>上手快：模型与相应优化都是以文本形式而非代码形式给出。
+Caffe给出了模型的定义、最优化设置以及预训练的权重，方便立即上手。</li>
+<li>速度快：能够运行最棒的模型与海量的数据。
+Caffe与cuDNN结合使用，测试AlexNet模型，在K40上处理每张图片只需要1.17ms.</li>
+<li>模块化：方便扩展到新的任务和设置上。
+可以使用Caffe提供的各层类型来定义自己的模型。</li>
+<li>开放性：公开的代码和参考模型用于再现。</li>
+<li>社区好：可以通过BSD-2参与开发与讨论。</li>
 </ul>
-<p>服务器上的python脚本</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding=utf-8</span></span><br><span class="line"><span class="keyword">from</span> flask <span class="keyword">import</span> Flask, request</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">import</span> sys</span><br><span class="line"><span class="keyword">from</span> pprint <span class="keyword">import</span> pprint</span><br><span class="line"><span class="keyword">import</span> hmac</span><br><span class="line">app = Flask(__name__)</span><br><span class="line"></span><br><span class="line">repo_path = <span class="string">&#x27;&#x27;</span> <span class="comment"># git仓库地址</span></span><br><span class="line">secret = <span class="string">&#x27;&#x27;</span>	   <span class="comment"># github配置的密钥</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">encryption</span>(<span class="params">data</span>):</span><br><span class="line">    key = secret.encode(<span class="string">&#x27;utf-8&#x27;</span>)</span><br><span class="line">    obj = hmac.new(key, msg=data, digestmod=<span class="string">&#x27;sha1&#x27;</span>)</span><br><span class="line">    <span class="keyword">return</span> obj.hexdigest()</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="meta">@app.route(<span class="params"><span class="string">&quot;/&quot;</span>, methods=[<span class="string">&quot;GET&quot;</span>, <span class="string">&quot;POST&quot;</span>]</span>)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">hello</span>():</span><br><span class="line">    post_data = request.data</span><br><span class="line">    token = encryption(post_data)</span><br><span class="line">    <span class="comment"># 认证签名是否有效</span></span><br><span class="line">    signature = request.headers.get(<span class="string">&#x27;X-Hub-Signature&#x27;</span>, <span class="string">&#x27;&#x27;</span>).split(<span class="string">&#x27;=&#x27;</span>)[-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">if</span> signature != token:</span><br><span class="line">        <span class="keyword">return</span> <span class="string">&#x27;failed&#x27;</span></span><br><span class="line"></span><br><span class="line">    os.system(<span class="string">&#x27;cd &#123;&#125; &amp;&amp; git pull&#x27;</span>.<span class="built_in">format</span>(repo_path))</span><br><span class="line">    <span class="keyword">return</span> <span class="string">&#x27;success&#x27;</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&quot;__main__&quot;</span>:</span><br><span class="line">    app.run(host=<span class="string">&quot;0.0.0.0&quot;</span>, port=port) <span class="comment"># 开启的port，记得打开对应端口的防火墙</span></span><br></pre></td></tr></table></figure>
-<p>若推送失败，可在服务器手动<code>git pull</code>，或者在<code>webhook</code>地址，<code>[Recent Deliveries]</code>中进行<code>Redeliver</code></p>
-]]></content>
-      <categories>
-        <category>Note</category>
-      </categories>
-      <tags>
-        <tag>Linux</tag>
-        <tag>Nginx</tag>
-        <tag>Github</tag>
-        <tag>Webhook</tag>
-        <tag>HTTPS</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>拉格朗日乘子法：从函数的求导到不等式约束的求解</title>
-    <url>/hexo_blog/2018/04/30/%E6%95%B0%E5%AD%A6/%E6%8B%89%E6%A0%BC%E6%9C%97%E6%97%A5%E4%B9%98%E5%AD%90%E6%B3%95%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/</url>
-    <content><![CDATA[<p>本篇文章简单梳理了关于拉格朗日乘子法的相关问题及一些相关概念的介绍。</p>
-<span id="more"></span>
-<h3 id="基本概念">基本概念</h3>
-<p>拉格朗日乘子法（Lagrange
-Multiplier）在某些地方又叫做拉格朗日乘数法。在求解最优化问题中，这个方法是最常用的方法之一。最优化问题往往抽象为某个函数，即求函数的最大值或最小值。求解最大值和最小值的求解可以相互转换（对偶）。对于求解最优化问题分为以下三种情况（难度依此递增）：</p>
-<h3 id="无约束条件">无约束条件</h3>
-<p>一般情况下，只需要对给出的函数进行求导，令其导函数为0的点可能就是极值点，将可能的结果分别代回原函数进行验证即可。</p>
-<p>假设有这样一个函数<span class="math inline">\(f(x)\)</span>，令<span
-class="math inline">\(g(x)=f^{&#39;}(x)\)</span>，<span
-class="math inline">\(g(x)=0\)</span>的解有<span
-class="math inline">\(x_1,x_2,x_3,\cdots\)</span>。求函数<span
-class="math inline">\(f(x)\)</span>的最优解就只需要分别将<span
-class="math inline">\(x_1,x_2,x_3,\cdots\)</span>代入<span
-class="math inline">\(f(x)\)</span>，比较大小，得到最优解<span
-class="math inline">\(x_i\)</span>。</p>
-<p>从这里开始引出<strong>拉格朗日乘子法</strong></p>
-<h3 id="等式约束条件">等式约束条件</h3>
-<p>设目标函数为<span class="math inline">\(f(x)\)</span>,
-约束条件为$h_k(x) $, <span
-class="math inline">\(k=1,2,\cdots,l\)</span>，即要求<span
-class="math inline">\(f(x)\)</span>的最优解，但是有<span
-class="math inline">\(l\)</span>个<span
-class="math inline">\(h(x)\)</span>等式约束（限制）求解。这里我们定义拉格朗日函数为</p>
-<p><span class="math display">\[
-F(x, \lambda)=f(x)+\sum\limits_{k=1}^{l}\lambda_kh_k(x)
-\]</span></p>
-<p><span
-class="math inline">\(\lambda_k\)</span>表示约束条件的待定系数。之后解变量的偏导方程，如下:</p>
-<p><span class="math display">\[
-\frac{\partial F}{\partial x_k}=0, \frac{\partial F}{\partial
-\lambda_k}=0, k=1,2,\cdots, l
-\]</span></p>
-<p>求解得到了<span
-class="math inline">\(l+1\)</span>个方程，将结果代回原方程验证即可得到解。</p>
-<p>举例：</p>
-<p>给定一个椭圆<span
-class="math inline">\(\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}=1\)</span>，求解<span
-class="math inline">\(f(x,y,z)=8xyz\)</span>的最大值。</p>
-<p>求解过程如下：</p>
+<h4 id="安装依赖">安装依赖</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># caffe 依赖</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler</span><br><span class="line"><span class="built_in">sudo</span> apt-get install --no-install-recommends libboost-all-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libatlas-base-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libhdf5-serial-dev</span><br><span class="line"><span class="comment"># Python 依赖</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-dev</span><br><span class="line"><span class="built_in">sudo</span> apt-get install libgflags-dev libgoogle-glog-dev liblmdb-dev</span><br></pre></td></tr></table></figure>
+<h4
+id="安装caffe可利用virtualenv新建一个纯净环境">安装caffe（可利用virtualenv新建一个纯净环境）</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 我们需要将caffe的源码下载下来，所以需要git</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install git</span><br><span class="line"><span class="comment"># 下载代码</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/BVLC/caffe.git</span><br><span class="line"><span class="comment"># 下载完成后，进入caffe文件夹</span></span><br><span class="line"><span class="built_in">cd</span> caffe</span><br><span class="line"><span class="comment"># 安装caffe的python依赖</span></span><br><span class="line"><span class="keyword">for</span> req <span class="keyword">in</span> $(<span class="built_in">cat</span> requirements.txt); <span class="keyword">do</span> pip install <span class="variable">$req</span>; <span class="keyword">done</span></span><br></pre></td></tr></table></figure>
+<h4 id="配置caffe">配置caffe</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 首先确保在caffe文件夹中，然后修改配置文件</span></span><br><span class="line"><span class="comment"># 进行备份</span></span><br><span class="line"><span class="built_in">cp</span> Makefile.config.example Makefile.config</span><br><span class="line"><span class="comment"># 进行编辑</span></span><br><span class="line">vim Makefile.config</span><br></pre></td></tr></table></figure>
 <ul>
-<li><p>通过拉格朗日乘子法的定义，得到如下方程</p>
-<p><span class="math display">\[
-\begin{array}{l}F(x,y,z,\lambda)&amp;=f(x,y,z)+\lambda\cdot h(x,y,z) \\
-&amp;=8xyz+\lambda\cdot(\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}-1)
-\end{array}
-\]</span></p></li>
-<li><p>接下来对<span
-class="math inline">\(F(x,y,z,\lambda)\)</span>求偏导，分别得到</p>
-<p><span class="math display">\[
-\frac{\partial F(x,y,z,\lambda)}{\partial x}=8yz+\frac{2\lambda
-x}{a^2}=0
-\]</span></p>
-<p><span class="math display">\[
-\frac{\partial F(x,y,z,\lambda)}{\partial y}=8xz+\frac{2\lambda
-y}{b^2}=0
-\]</span></p>
-<p><span class="math display">\[
-\frac{\partial F(x,y,z,\lambda)}{\partial z}=8xy+\frac{2\lambda
-z}{c^2}=0
-\]</span></p>
-<p><span class="math display">\[
-\frac{\partial F(x,y,z,\lambda)}{\partial
-\lambda}=\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}-1=0
-\]</span></p></li>
-<li><p>联立前三个方程可以得到<span
-class="math inline">\(bx=ay\)</span>和<span
-class="math inline">\(az=cx\)</span>，代入最后一个方程，解得：</p>
-<p><span class="math display">\[
-x=\frac{\sqrt{3}}{3}a
-\]</span></p>
-<p><span class="math display">\[
-y=\frac{\sqrt{3}}{3}b
-\]</span></p>
-<p><span class="math display">\[
-z=\frac{\sqrt{3}}{3}c
-\]</span></p></li>
-<li><p>最后代入原方程，解得最大体积</p>
-<p><span class="math display">\[
-V_{max}=f(\frac{\sqrt{3}}{3}a,\frac{\sqrt{3}}{3}b,\frac{\sqrt{3}}{3}c)=\frac{8\sqrt{3}}{9}abc
-\]</span></p></li>
+<li>因为CPU MODE, 所以在CPU_ONLY := 1前面的#要去掉</li>
+<li>两个路径要改成这样:(添加后面的两个hdf5的路径,
+否则编译时报hdf5错误)</li>
 </ul>
-<h3 id="不等式约束条件">不等式约束条件</h3>
-<p>设目标函数为<span class="math inline">\(f(x)\)</span>,
-不等式约束条件为<span class="math inline">\(c_i(x)\)</span>,<span
-class="math inline">\(i=1,2,\cdots,k\)</span>，等式约束条件为$h_j(x) $,
-<span class="math inline">\(j=1,2,\cdots,l\)</span>，即要求<span
-class="math inline">\(f(x)\)</span>的最优解，但是有<span
-class="math inline">\(k\)</span>个<span
-class="math inline">\(g(x)\)</span>不等式和<span
-class="math inline">\(l\)</span>个<span
-class="math inline">\(h(x)\)</span>等式约束（限制）求解。这里我们定义拉格朗日函数为</p>
-<p><span class="math display">\[
-L(x,a,b)=f(x)+\sum\limits^{k}_{i=1}a_ic_i(x)+\sum\limits^{l}_{j=1}b_jh_j(x)
-\]</span></p>
-<p>这里用一个叫做KKT条件的方法，把所有的不等式约束、等式约束和目标函数全部写为一个式子<span
-class="math inline">\(L(a,b,x)=f(x)+a*c(x)+b*h(x)\)</span>。</p>
-<p>KKT条件是指最优解必须满足以下条件：</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># Whatever else you find you need goes here.</span></span><br><span class="line">INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial</span><br><span class="line">LIBRARY_DIRS := $(PYTHON_LIB) /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu/hdf5/serial</span><br></pre></td></tr></table></figure>
+<p>修改完成之后，保存退出即可</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进行编译</span></span><br><span class="line">make pycaffe</span><br><span class="line">make all</span><br><span class="line">make <span class="built_in">test</span></span><br><span class="line">make runtest</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译结束后，需要设置环境变量</span></span><br><span class="line"><span class="comment"># 首先确保已经进入到caffe文件夹中，之后查看当前路径</span></span><br><span class="line"><span class="built_in">pwd</span></span><br><span class="line"><span class="comment"># 假设输出结果是xxx，那么将xxx复制下来，黏贴至下面xxx的位置</span></span><br><span class="line"><span class="built_in">export</span> PYTHONPATH=xxx/python:<span class="variable">$PYTHONPATH</span></span><br></pre></td></tr></table></figure>
+<h4 id="测试">测试</h4>
+<p>一般来说，如果运气好的话，到这里基本就算安装完成，但我们还是要测试一下</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 进入python环境</span></span><br><span class="line">python</span><br><span class="line"><span class="comment"># 导入caffe模块</span></span><br><span class="line">&gt;caffe</span><br><span class="line"><span class="comment"># 如果此处没有报任何错误，恭喜你安装成功</span></span><br></pre></td></tr></table></figure>
+<h4 id="部分报错解决方案">部分报错解决方案</h4>
+<p><strong>最好的方案莫过于，复制报错代码，Google一下 </strong>
+下面提供几种，我安装时的几处坑</p>
 <ul>
-<li><span class="math inline">\(L(a,b,x)\)</span>求<span
-class="math inline">\(x\)</span>求导为0</li>
-<li><span class="math inline">\(h(x)=0\)</span></li>
-<li><span class="math inline">\(a*c(x)=0\)</span></li>
+<li>设置环境变量，注意一定是在caffe文件夹下，路径是caffe/python的绝对路径</li>
+<li>安装依赖报错，这个没什么好讲，直接google吧</li>
+<li>make的时候报错，想想看是否编辑了Makefile.config。之后make
+clean一下，再重新编译一下</li>
+<li>CPU_ONLY这个选项为1时，需要修改一下<code>caffe/examples/mnist/lenet_solver.prototxt</code>，将其中的<code>solver_mode:GPU</code>改为CPU</li>
 </ul>
-<p>这样就可以将不等式约束转换为了等式约束，求<span
-class="math inline">\(L(a,b,x)\)</span>的偏导等于零即可求得最优参数。</p>
-<p><span class="math display">\[
-\min f(x)\Leftrightarrow \min\limits_x \max\limits_{a,b} L(x,a,b)
-\]</span></p>
-<p>经过对偶变换得：</p>
-<p><span class="math display">\[
-\max\limits_{a,b} \min L(x,a,b)
-\]</span></p>
-<p>因为<span class="math inline">\(c(x)&lt;0\)</span>，所以只有当<span
-class="math inline">\(a*c(x)=0\)</span>时，<span
-class="math inline">\(L(x,a,b)\)</span>才能取得最大值。以上就可以使用同等式约束的求解方法进行求解。</p>
-<h3 id="补充部分">补充部分</h3>
-<h4 id="定义不等式约束问题的标准写法">定义不等式约束问题的标准写法</h4>
-<p><span class="math inline">\(f(x), c_i(x),h_j(x)\)</span>是定义在<span
-class="math inline">\(R^n\)</span>上的连续可微函数，考虑约束最优化问题：</p>
-<p><span class="math inline">\(\min f(x)  \quad  x\in R^n\)</span></p>
-<p><span class="math inline">\(s.t. c_i(x) \leq 0 \quad i =
-1...k\)</span></p>
-<pre><code>$h_j(x)=0 \quad j = 1...l$</code></pre>
-<p>即有<span class="math inline">\(k\)</span>个不等式约束<span
-class="math inline">\(c_i(x)\)</span>和<span
-class="math inline">\(l\)</span>个等式约束<span
-class="math inline">\(h_j(x)\)</span></p>
-<h4 id="拉格朗日函数">拉格朗日函数</h4>
-<p><span class="math inline">\(L(x,\alpha,
-\beta)=f(x)+\sum^{k}_{i=1}\alpha_ic_i(x)+\sum^{l}_{j=1}\beta_jh_j(x)\)</span></p>
-<p>其中的<span class="math inline">\(\alpha_i\)</span>和<span
-class="math inline">\(\beta_j\)</span>，称为<strong>拉格朗日乘子</strong>，<span
-class="math inline">\(\alpha_i \geq 0\)</span></p>
-<h4 id="几何意义">几何意义</h4>
-<p>对于一个二维空间来说，求解最短路径的问题。将维度升高，变为对于多维空间来说，求解最短路径问题。</p>
-<p>比较直观的解释，爬山的目标是到山顶，根据梯度上升（下降）的方法来寻求优解，即爬到山顶。</p>
-<h4 id="对偶概念的简单介绍">对偶概念的简单介绍</h4>
-<p>一对一的一种方式，把一种概念、公理或数学结构转换为另一种概念、公理或数学结构。如果A的对偶是B，那么B的对偶是A。A的对偶有时候是A的本身。</p>
-<p>有界线性泛函数：复杂对象映射到<strong>简单对象</strong>上</p>
-<p>群表示：简单对象具有类似线性结构的<strong>简单结构</strong></p>
-<p>范畴论：映射箭头所构成的集合</p>
-<p>（像集，映射）——&gt;原像集</p>
-<p>对联可以称为一种对偶，上联与下联有对应的关系（相似和取逆）。对偶是有一定要有自然合理的对应关系（或者说能够吹出来的自然合理关系）。</p>
-<p>常见的自然、合理法则首推：相似（包括同构）和取逆（包括互反）</p>
-<p>常见对偶的例子：</p>
-<ol type="1">
-<li>奇数和偶数</li>
-<li>洛必达法则和Stolz 定理（相当于离散洛必达法则）</li>
-<li>Fourier 系数与 Fourier 变换（傅里叶）</li>
-</ol>
-<p>对偶可以理解为，一个问题不好理解转换为另一个简单方式的问题来理解，二者在本质上是相同的。</p>
-<p>比如：某个函数的最大化不方便求解，那么就求这个函数的最小化。</p>
-<p>未完待续……</p>
-<ol type="1">
-<li>罚函数</li>
-<li>广义乘子法</li>
-<li>KKT的求导证明</li>
-</ol>
+<p><a
+href="http://blog.csdn.net/sinat_26917383/article/details/53502719">参考文章</a></p>
+<h3 id="实现前准备">实现前准备</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 默认已安装git，未安装请sudo apt get install git</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/fzliu/style-transfer</span><br><span class="line"></span><br><span class="line"><span class="comment"># pycaffe环境布置</span></span><br><span class="line"><span class="built_in">sudo</span> pip install progressbar</span><br><span class="line"></span><br><span class="line"><span class="comment"># 下载训练模型，这里推荐vgg16</span></span><br><span class="line"><span class="comment"># 方法一，使用scripts/ 下的download_models.sh这个方法可能很慢。。。。</span></span><br><span class="line">bash scripts/download_models.sh vgg16</span><br><span class="line"><span class="comment"># 方法二</span></span><br><span class="line">wget http://www.robots.ox.ac.uk/~vgg/software/very_deep/caffe/VGG_ILSVRC_16_layers.caffemodel</span><br></pre></td></tr></table></figure>
+<p>使用方法二下载的model需要放在./model/vgg16下，使用方法一下载的已经默认放在那</p>
+<h3 id="实现">实现</h3>
+<p>确保在源码文件夹中</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 格式</span></span><br><span class="line">python style.py -s &lt;style_image&gt; -c &lt;content_image&gt; -m &lt;model_name&gt; -g 0</span><br><span class="line"></span><br><span class="line"><span class="comment"># 举个栗子，model_name直接写vgg16或其他，不需要加具体路径</span></span><br><span class="line">python style.py -s images/style/starry_night.jpg -c images/content/nanjing.jpg -m vgg16 -g -1</span><br></pre></td></tr></table></figure>
+<p>提示：CPU很慢，真的很慢，估计要等一会才会有时间显示出来</p>
+<p>参数解析：</p>
+<ul>
+<li>-s, 风格图位置；</li>
+<li>-c, 内容图位置；</li>
+<li>-m, 模型位置；</li>
+<li>-g, 什么模式，-1为CPU，0为单个GPU，1为两个GPU。</li>
+</ul>
+<h3 id="调整参数">调整参数</h3>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 确保在源码文件夹下,修改style.py配置文件</span></span><br><span class="line">vim style.py</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 修改文件大小，改为1024</span></span><br><span class="line">parser.add_argument(<span class="string">&quot;-l&quot;</span>, <span class="string">&quot;--length&quot;</span>, default=1024, <span class="built_in">type</span>=<span class="built_in">float</span>, required=False, <span class="built_in">help</span>=<span class="string">&quot;maximum image length&quot;</span>)</span><br><span class="line">def transfer_style(self, img_style, img_content, length=1024, ratio=1e5,</span><br><span class="line">n_iter=512, init=<span class="string">&quot;-1&quot;</span>, verbose=False, callback=None)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 修改迭代次数，个人认为400就差不多了，可自行修改；50次迭代之后改变的就是背景纹理，按需更改</span></span><br><span class="line">parser.add_argument(<span class="string">&quot;-n&quot;</span>, <span class="string">&quot;--num-iters&quot;</span>, default=400, <span class="built_in">type</span>=int, required=False, <span class="built_in">help</span>=<span class="string">&quot;L-BFGS iterations&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 还有很多其他可以修改的地方，在此就不一一赘述了</span></span><br></pre></td></tr></table></figure>
+<h2
+id="fast-neural-style-tensorflow的实现">fast-neural-style-tensorflow的实现</h2>
+<p><strong>环境介绍 </strong></p>
+<p>Ubuntu16.04 + CPU + python2.7 + tensorflow 1.0</p>
+<h3 id="搭建tensorflow环境">搭建tensorflow环境</h3>
+<p>环境说明：Ubuntu16.04 注：tensorflow只支持64位系统</p>
+<h4 id="安装前准备">安装前准备</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 确保有pip工具，一般自带，没有的话请按下面这条命令进行安装</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-pip python-dev</span><br></pre></td></tr></table></figure>
+<h4 id="cpu安装">CPU安装</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># python2</span></span><br><span class="line"><span class="built_in">sudo</span> pip installl tensorflow</span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line"><span class="built_in">sudo</span> pip3 install tensorflow</span><br></pre></td></tr></table></figure>
+<h4 id="gpu安装">GPU安装</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 安装地址请根据自己需求进行声明环境变量</span></span><br><span class="line"><span class="comment"># python2</span></span><br><span class="line"><span class="built_in">export</span> TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.12.0rc0-cp27-none-linux_x86_64.whl</span><br><span class="line"></span><br><span class="line"><span class="comment"># python3</span></span><br><span class="line"><span class="built_in">export</span> TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-0.12.0rc0-cp35-cp35m-linux_x86_64.whl</span><br><span class="line"></span><br><span class="line"><span class="comment"># Python2</span></span><br><span class="line"><span class="built_in">sudo</span> pip install --upgrade <span class="variable">$TF_BINARY_URL</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># Python3</span></span><br><span class="line"><span class="built_in">sudo</span> pip3 install --upgrade <span class="variable">$TF_BINARY_URL</span></span><br></pre></td></tr></table></figure>
+<p><a
+href="https://morvanzhou.github.io/tutorials/machine-learning/tensorflow/1-2-install/">参考文章</a></p>
+<h4 id="源码安装">源码安装</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 下载最新的tensorflow项目</span></span><br><span class="line">git <span class="built_in">clone</span> https://github.com/tensorflow/tensorflow</span><br><span class="line"><span class="comment"># 进入tensorflow</span></span><br><span class="line"><span class="built_in">cd</span> tensorflow</span><br><span class="line"><span class="comment"># 切换到需要安装的版本分支，以1.0为栗子</span></span><br><span class="line">git checkout r1.0</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装bazel</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">echo</span> <span class="string">&quot;deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8&quot;</span> | <span class="built_in">sudo</span> <span class="built_in">tee</span> /etc/apt/sources.list.d/bazel.list</span><br><span class="line"></span><br><span class="line">curl https://bazel.build/bazel-release.pub.gpg | <span class="built_in">sudo</span> apt-key add -</span><br><span class="line"></span><br><span class="line"><span class="built_in">sudo</span> apt-get update &amp;&amp; <span class="built_in">sudo</span> apt-get install bazel</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装依赖</span></span><br><span class="line"><span class="comment"># Python 2</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python-numpy python-dev python-pip python-wheel</span><br><span class="line"></span><br><span class="line"><span class="comment"># Python 3</span></span><br><span class="line"><span class="built_in">sudo</span> apt-get install python3-numpy python3-dev python3-pip python3-wheel</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行编译，确保在tensorflow文件夹中</span></span><br><span class="line">./configure</span><br><span class="line"><span class="comment"># 提示信息可以一路回车</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译CPU版本</span></span><br><span class="line">bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package</span><br><span class="line"><span class="comment"># 编译GPU版本</span></span><br><span class="line">bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package</span><br><span class="line"></span><br><span class="line"><span class="comment"># 编译生成的whl包</span></span><br><span class="line">bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg</span><br><span class="line"></span><br><span class="line"><span class="comment"># 安装生成的包</span></span><br><span class="line"><span class="built_in">sudo</span> pip install /tmp/tensorflow_pkg/tensorflow*.whl</span><br></pre></td></tr></table></figure>
+<p><a
+href="https://tensorflow.feisky.xyz/install/src.html">参考文章</a></p>
+<h4 id="测试-1">测试</h4>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 打开对应版本的python终端</span></span><br><span class="line">&gt;import tensorflow</span><br><span class="line"><span class="comment"># 未报错即安装成功</span></span><br></pre></td></tr></table></figure>
+<h3 id="实现前准备-1">实现前准备</h3>
+<p>下载已经训练好的模型</p>
+<p><a
+href="https://pan.baidu.com/s/1i4GTS4d#list/path=%2F">百度云盘链接</a></p>
+<figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line"># 安装pyyaml</span><br><span class="line">sudo pip install pyyaml</span><br></pre></td></tr></table></figure>
+<h3 id="实现-1">实现</h3>
+<p>确保在源码文件夹中</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 格式</span></span><br><span class="line">python eval.py --model_file &lt;path of ckpt-done&gt; --image_file &lt;path of image&gt;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 举个栗子,假设wave.ckpt-done放在了源码文件夹中的models文件夹中</span></span><br><span class="line">python eval.py --model_file models/wave.ckpt-done --image_file img/test.jpg</span><br><span class="line"><span class="comment"># 默认生成位置在generated，默认生成文件名称为res.jpg。可自行在eval.py中更改</span></span><br></pre></td></tr></table></figure>
+<h3 id="训练新的风格模型">训练新的风格模型</h3>
+<p><strong>下载</strong></p>
+<p><a
+href="http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz">vgg16训练模型</a>(500多M)</p>
+<p><a
+href="http://msvocds.blob.core.windows.net/coco2014/train2014.zip">训练数据集</a>(12.6G)</p>
+<p>或者使用wget命令下载</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># vgg16训练模型</span></span><br><span class="line">wget http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练数据集</span></span><br><span class="line">wget http://msvocds.blob.core.windows.net/coco2014/train2014.zip</span><br></pre></td></tr></table></figure>
+<p><strong>移动模型到对应目录中</strong></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 在源码文件夹中新建文件夹</span></span><br><span class="line"><span class="built_in">mkdir</span> pretrained</span><br><span class="line"></span><br><span class="line"><span class="comment"># 下载完成之后需要进行解压</span></span><br><span class="line">tar -zxvf vgg_16_2016_08_28.tar.gz</span><br><span class="line"></span><br><span class="line"><span class="comment"># 移动文件夹到pretrained</span></span><br><span class="line"><span class="built_in">cp</span> &lt;path of vgg16&gt; &lt;pretrained&gt;</span><br></pre></td></tr></table></figure>
+<p><strong>训练新的模型</strong></p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 解压数据集</span></span><br><span class="line">unzip train2014.zip</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入数据集</span></span><br><span class="line"><span class="built_in">cd</span> train2014</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建立软链接，方便训练命令的输入</span></span><br><span class="line"><span class="built_in">ln</span> -s &lt;train2014的绝对路径&gt; train2014</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进入源码文件夹中</span></span><br><span class="line"><span class="comment"># 开始训练, wave.yml是作者预先配置好的文件，如果训练自己的模型，需要自己新写一份yml文件</span></span><br><span class="line">python train.py -c conf/wave.yml</span><br></pre></td></tr></table></figure>
+<h2 id="总结">总结</h2>
+<p>如果只是单纯的想玩玩风格转换的话，可以试试fast-neural-style-tensorflow，不过训练新模型还是不建议（特殊需要除外），耗时耗力</p>
+<p>如果想锻炼自己，从零开始学习，可以从读论文开始，然后试试style-transfer代码</p>
+<p><a
+href="http://blog.csdn.net/sinat_26917383/article/details/53978519">style-transfer
+参考文章</a></p>
+<p><a href="https://github.com/fzliu/style-transfer">style-transfer
+代码</a></p>
+<p><a
+href="https://github.com/hzy46/fast-neural-style-tensorflow/blob/master/README.md#use-trained-models">fast-neural-style-tensorflow
+代码</a></p>
 ]]></content>
       <categories>
-        <category>MachineLearning</category>
+        <category>Environment</category>
       </categories>
       <tags>
-        <tag>数学</tag>
-        <tag>Optimization</tag>
-        <tag>拉格朗日乘子法</tag>
+        <tag>Python</tag>
+        <tag>Tensorflow</tag>
+        <tag>Caffe</tag>
+        <tag>风格迁移</tag>
       </tags>
   </entry>
   <entry>
-    <title>最优化算法学习记录</title>
-    <url>/hexo_blog/2021/02/03/%E6%95%B0%E5%AD%A6/%E6%9C%80%E4%BC%98%E5%8C%96%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/</url>
-    <content><![CDATA[<p>本文提供了优化问题的概述，包括背景，分类，以及线性，非线性和二次规划问题的求解方法。本文还讨论了全局优化，并提供了非凸函数和凸函数的例子，以及线性规划和二次规划的例子。</p>
+    <title>NLTK阅读笔记Ⅱ</title>
+    <url>/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/</url>
+    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
+<p>接 <a href="/hexo_blog/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/" title="NLTK阅读笔记Ⅰ">NLTK阅读笔记Ⅰ</a></p>
 <span id="more"></span>
-<p>优化问题的最终目的就是求最小值（最大值转为求最小值）</p>
-<h3 id="简单划分">简单划分</h3>
-<ol type="1">
-<li>解析法</li>
-<li><strong>迭代法</strong>
-<ol type="1">
-<li>常规方法</li>
-<li>技巧
+<h2 id="第三章加工原料文本">第三章：加工原料文本</h2>
+<p><a href="http://www.gutenberg.org/catalog/">古腾堡项目</a></p>
+<p>可以从这个项目中获取感兴趣的文本，之后进行分析。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="comment"># 将原始文本字符串转换为list，以\r\n标点符号等为分隔符进行切割</span></span><br><span class="line">tokens = nltk.word_tokenize(raw)</span><br><span class="line"><span class="comment"># 创建NLTK文本</span></span><br><span class="line">text = nltk.Text(tokens)</span><br></pre></td></tr></table></figure>
+<p>处理文本部分：</p>
+<p><code>requests</code>： 数据获取</p>
+<p><code>BeautifulSoup</code>、<code>re</code>、<code>xpath</code>：
+文本处理</p>
+<p><code>feedparser</code>：处理RSS订阅</p>
+<p><code>os</code>：处理本地文件</p>
+<p>字符串基本操作在这里请见下图。注：字符串是不可改变的。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-2.png" /></p>
+<p>关于编码这个大坑，在Python3中做的已经比Python2好太多，在这里也不做介绍了。介绍一些关于编码处理的库。</p>
+<p><code>codecs</code>、<code>unicodedata</code></p>
+<p>正则表达式部分，推荐自主学习。难度系数不算高。搜索关键词，Python,
+正则表达式,
+re即可。值得一提的是，<code>re</code>是目前我了解处理文本速度最快最优雅的一种方式。</p>
+<p>英文文本是高度冗余的，忽略掉词内部的元音仍然可以很容易的阅读，有些时候这很明显。例如：declaration
+变成 dclrtn，inalienable 变成 inlnble，保留所有词首或词尾的元音序列
+。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"></span><br><span class="line">regexp = <span class="string">r&#x27;^[AEIOUaeiou]+|[AEIOUaeiou]+$|[^AEIOUaeiou]&#x27;</span></span><br><span class="line"><span class="comment"># 压缩元音字符串</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compress</span>(<span class="params">word</span>):</span><br><span class="line">	pieces = re.findall(regexp, word)</span><br><span class="line">    <span class="keyword">return</span> <span class="string">&#x27;&#x27;</span>.join(pieces)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取语料库数据</span></span><br><span class="line">english_udhr = nltk.corpus.udhr.words(<span class="string">&#x27;English-Latin1&#x27;</span>)</span><br><span class="line"><span class="comment"># 进行压缩</span></span><br><span class="line">nltk.tokenwrap(compress(w) <span class="keyword">for</span> w <span class="keyword">in</span> english_udhr[:<span class="number">75</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则结合条件频率，提取辅音-元音序列</span></span><br><span class="line">rotokas_words = nltk.corpus.toolbox.words(<span class="string">&#x27;rotokas.dic&#x27;</span>)</span><br><span class="line">cvs = [cv <span class="keyword">for</span> w <span class="keyword">in</span> rotokas_words <span class="keyword">for</span> cv <span class="keyword">in</span> re.findall(<span class="string">r&#x27;[ptksvr][aeiou]&#x27;</span>, w)]</span><br><span class="line">cfd = nltk.ConditionalFreqDist(cvs)</span><br><span class="line">cfd.tabulate()</span><br><span class="line"></span><br><span class="line">    a   e   i   o   u</span><br><span class="line">k <span class="number">418</span> <span class="number">148</span>  <span class="number">94</span> <span class="number">420</span> <span class="number">173</span></span><br><span class="line">p  <span class="number">83</span>  <span class="number">31</span> <span class="number">105</span>  <span class="number">34</span>  <span class="number">51</span></span><br><span class="line">r <span class="number">187</span>  <span class="number">63</span>  <span class="number">84</span>  <span class="number">89</span>  <span class="number">79</span></span><br><span class="line">s   <span class="number">0</span>   <span class="number">0</span> <span class="number">100</span>   <span class="number">2</span>   <span class="number">1</span></span><br><span class="line">t  <span class="number">47</span>   <span class="number">8</span>   <span class="number">0</span> <span class="number">148</span>  <span class="number">37</span></span><br><span class="line">v  <span class="number">93</span>  <span class="number">27</span> <span class="number">105</span>  <span class="number">48</span>  <span class="number">49</span></span><br></pre></td></tr></table></figure>
+<h3 id="词干提取器">词干提取器</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查找词干，laptop与laptops其实只是单复数的区别。词干是相同的</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">stem</span>(<span class="params">word</span>):</span><br><span class="line">    regexp = <span class="string">r&#x27;^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)?$&#x27;</span></span><br><span class="line">    stem, suffix = re.findall(regexp, word)[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">return</span> stem</span><br><span class="line"></span><br><span class="line"><span class="comment"># nltk自带的词干提取器</span></span><br><span class="line"><span class="comment"># 索引一些文本和使搜索支持不同词汇形式的话，使用Porter词干提取器</span></span><br><span class="line">porter = nltk.PorterStemmer()</span><br><span class="line">lancaster = nltk.LancasterStemmer()</span><br><span class="line"><span class="comment"># 编译一些文本的词汇或想要一个有效词条（或中心词）列表，使用WordNet词形归并器</span></span><br><span class="line">wnl = nltk.WordNetLemmatizer()</span><br></pre></td></tr></table></figure>
+<h3 id="分词">分词：</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用正则进行分词</span></span><br><span class="line">re.findall(<span class="string">r&quot;\w+(?:[-&#x27;]\w+)|&#x27;|[-.(]+|\S\w&quot;</span>, raw)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用nltk自带的函数进行分词，效率更高且不需要特殊处理符号</span></span><br><span class="line">nltk.regexp_tokenize()</span><br><span class="line"></span><br><span class="line">pattern = <span class="string">r&#x27;&#x27;&#x27;(?x) # 去除嵌入的空白字符和注释</span></span><br><span class="line"><span class="string">			([A-Z]\.)+ # 设定允许的标识符（单词）</span></span><br><span class="line"><span class="string">			| \w+(-\w+)* # 可选的内部连字符</span></span><br><span class="line"><span class="string">			| \$?\d+(\.\d+)?%? # 货币与百分比</span></span><br><span class="line"><span class="string">			| \.\.\. # 省略符</span></span><br><span class="line"><span class="string">			| [][.,;&quot;&#x27;?():-_`] # 额外的标志</span></span><br><span class="line"><span class="string">		&#x27;&#x27;&#x27;</span></span><br><span class="line">nltk.regexp_tokenize(text, pattern)</span><br></pre></td></tr></table></figure>
+<p>已经分词好的数据举例：</p>
+<p>《华尔街日报》原始文本（<code>nltk.corpus.treebank_raw.raw()</code>）和分好词的版本（<code>nltk.corpus.treebank.words()</code>）</p>
+<p>分词的最后一个问题是缩写的存在，如“didn't”。如果我们想分析一个句子的意思，将这种形式规范化为两个独立的形式：“did”和“n't”(不是
+not)可能更加有用。我们可以通过查表来做这项工作。</p>
+<p>链表与字符串部分不做介绍，是Python字符串的常规操作</p>
+<p>补充部分：</p>
+<ul>
+<li><p>建立词汇表流程</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-1.png" /></p></li>
+<li><p>编码点：每个字符分配一个编号</p></li>
+<li><p>解码：通过一些机制来将文本翻译成 Unicode的过程</p></li>
+<li><p>编码：将Unicode转化为其它编码的过程</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-3.png" /></p></li>
+<li><p>从 Unicode的角度来看，字符是可以实现一个或多个
+字形的抽象的实体。只有字形可以出现在屏幕上或被打印在纸上。一个字体是一个字符到字形映射。</p></li>
+<li><p>规范化文本：如将所有的文本小写保存，使用<code>lower()</code>这个函数</p></li>
+<li><p>​断句：在将文本分词之前，我们需要将它分割成句子</p></li>
+<li><p>分词（重点内容）：</p>
 <ol type="1">
-<li>切平面</li>
-<li>椭圆法</li>
-<li>内点法</li>
-</ol></li>
+<li><p>找到一种方法来分开文本内容与分词标志。给每个字符标注一个布尔值来指示这个字符后面是否有一个分词标志</p></li>
+<li><p>找到将文本字符串正确分割成词汇的字位串。根据一个字典，来根据字典中词的序列来重构源文本。定义一个目标函数，通过字典大小与重构源文本所需的信息量尽力优化它的值。</p></li>
+<li><p>找到最大化目标函数值的0和1的模式。使用<strong>模拟退火算法</strong>的非确定性搜索。</p></li>
 </ol></li>
-</ol>
-<h3 id="background">Background</h3>
-<ol type="1">
-<li>目标函数、约束、线性（非线性）</li>
-<li>凸优化问题（是否为凸问题）</li>
-<li>局部最优、全局最优</li>
-<li>矩阵求导（梯度），矩阵的一阶导数、二阶导数（Hessian）</li>
-<li>解析与迭代</li>
-</ol>
-<h3 id="how">How</h3>
-<ol type="1">
-<li>目标函数、约束均为线性：单纯形法。大问题：内点法</li>
-<li>目标函数二次，约束线性：改进单纯形法。大问题：内点法</li>
-<li>凸问题：切割平面、椭球算法、<strong>内点法</strong></li>
-<li>目标函数存在多个局部最小值，全局优化算法（启发式算法/智能优化算法）</li>
-<li>非线性非凸问题（最难），梯度和海森信息（二阶导）-&gt;构造解析表达式</li>
-<li>无约束问题
+</ul>
+<h2 id="第四章编写结构化程序">第四章：编写结构化程序</h2>
+<p>大量关于Python代码，不做介绍。此部分适宜阅读人群，使用Python时间不超过一年（不够熟练）</p>
+<p>文档说明模块：<code>docstring</code></p>
+<p>调试技术：<code>pdb</code></p>
+<p>绘图：<code>matplotlib</code></p>
+<p>图的绘制：<code>networkx</code></p>
+<p>数据统计表：<code>csv</code>（这里个人推荐有使用<code>pandas</code>，虽然它很”重“）</p>
+<p>数值运算：<code>numpy</code></p>
+<p>算法设计：</p>
 <ol type="1">
-<li>Levenberg-Marquardt</li>
-<li>牛顿算法</li>
-<li>拟牛顿</li>
-<li>最速下降</li>
-<li>Powell</li>
-</ol></li>
-<li>线性约束：SQP、梯度投影法</li>
-<li>非线性约束：SQP、转为为无约束</li>
+<li>递归</li>
+<li>权衡空间与时间</li>
+<li>动态规划</li>
 </ol>
-<h3 id="优化问题分类">优化问题分类</h3>
-<ol type="1">
-<li><p>无约束问题（非线性最小二乘问题）</p>
-<ol type="1">
-<li>牛顿法</li>
-<li>逆牛顿法</li>
-<li>Levenberg-Marquardt算法</li>
-<li>Nelder-Mead 方法</li>
-<li>其他
-<ol type="1">
-<li>搜索方向和一维搜索法</li>
-</ol></li>
-</ol></li>
-<li><p>约束问题</p>
-<ol type="1">
-<li>规划问题
-<ol type="1">
-<li>线性规划（整数规划）
-<ol type="1">
-<li>单纯形解法</li>
-</ol></li>
-<li>二次规划
-<ol type="1">
-<li>改进单纯形法</li>
-</ol></li>
-</ol></li>
-<li>非线性优化的约束
-<ol type="1">
-<li>等式约束
-<ol type="1">
-<li>线性等式约束</li>
-<li>非线性等式约束</li>
-</ol></li>
-<li>不等式约束
+<h2 id="第五章分类和标注词汇">第五章：分类和标注词汇</h2>
+<p>NLP基本技术：</p>
+<ul>
+<li>序列标注</li>
+<li>N-gram 模型</li>
+<li>回退和评估</li>
+</ul>
+<p>将词汇按它们的<strong>词性</strong>（parts-of-speech，POS）分类以及相应的标注它们的过程被称为<strong>词性标注</strong>（part-of-speech
+tagging, POS
+tagging）或干脆简称<strong>标注</strong>。词性也称为<strong>词类</strong>或<strong>词汇范畴</strong>。用于特定任务的标记的集合被称为一个<strong>标记集</strong>。</p>
+<h3 id="词性标注器">词性标注器</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line">text = nltk.word_tokenize(<span class="string">&quot;And now for something completely different&quot;</span>)</span><br><span class="line"><span class="comment"># 标注出text中每个词的词性</span></span><br><span class="line">nltk.pos_tag(text)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">[(&#x27;And&#x27;, &#x27;CC&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;now&#x27;, &#x27;RB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;for&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;something&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;completely&#x27;, &#x27;RB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;different&#x27;, &#x27;JJ&#x27;)]</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line">text = nltk.Text(word.lower() <span class="keyword">for</span> word <span class="keyword">in</span> nltk.corpus.brown.words())</span><br><span class="line"><span class="comment"># 为一个词找出所有它的上下文</span></span><br><span class="line">text.similar(<span class="string">&#x27;woman&#x27;</span>)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">man time day year car moment world house family child country boy</span></span><br><span class="line"><span class="string">state job place way war girl work word</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br></pre></td></tr></table></figure>
+<h3 id="标注语料库">标注语料库</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建一个有标识的字符串</span></span><br><span class="line">tagged_token = nltk.tag.str2tuple(<span class="string">&#x27;fly/NN&#x27;</span>)</span><br><span class="line">tagged_token</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">tagged_token</span></span><br><span class="line"><span class="string">(&#x27;fly&#x27;, &#x27;NN&#x27;)</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line">tagged_token[<span class="number">0</span>]</span><br><span class="line">tagged_token[<span class="number">1</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 批量标注</span></span><br><span class="line">sent = <span class="string">&#x27;&#x27;&#x27;The/AT grand/JJ jury/NN commented/VBD on/IN a/AT number/NN of/INother/AP topics/NNS ,/, AMONG/IN them/PPO the/AT Atlanta/NP and/CCFulton/NP-tl County/NN-tl purchasing/VBG departments/NNS which/WDT it/PPsaid/VBD ``/`` ARE/BER well/QL operated/VBN and/CC follow/VB generally/Raccepted/VBN practices/NNS which/WDT inure/VB to/IN the/AT best/JJTinterest/NN of/IN both/ABX governments/NNS &#x27;&#x27;/&#x27;&#x27; ./.&#x27;&#x27;&#x27;</span></span><br><span class="line">[nltk.tag.str2tuple(t) <span class="keyword">for</span> t <span class="keyword">in</span> sent.split()]</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">[(&#x27;The&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;grand&#x27;, &#x27;JJ&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;jury&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;commented&#x27;, &#x27;VBD&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;on&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;a&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;number&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;of/INother&#x27;, &#x27;AP&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;topics&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;,&#x27;, &#x27;,&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;AMONG&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;them&#x27;, &#x27;PPO&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;the&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;Atlanta&#x27;, &#x27;NP&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;and/CCFulton&#x27;, &#x27;NP-TL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;County&#x27;, &#x27;NN-TL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;purchasing&#x27;, &#x27;VBG&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;departments&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;which&#x27;, &#x27;WDT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;it/PPsaid&#x27;, &#x27;VBD&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;``&#x27;, &#x27;``&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;ARE&#x27;, &#x27;BER&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;well&#x27;, &#x27;QL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;operated&#x27;, &#x27;VBN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;and&#x27;, &#x27;CC&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;follow&#x27;, &#x27;VB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;generally/Raccepted&#x27;, &#x27;VBN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;practices&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;which&#x27;, &#x27;WDT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;inure&#x27;, &#x27;VB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;to&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;the&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;best/JJTinterest&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;of&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;both&#x27;, &#x27;ABX&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;governments&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&quot;&#x27;&#x27;&quot;, &quot;&#x27;&#x27;&quot;),</span></span><br><span class="line"><span class="string"> (&#x27;.&#x27;, &#x27;.&#x27;)]</span></span><br><span class="line"><span class="string"> &#x27;&#x27;&#x27;</span></span><br></pre></td></tr></table></figure>
+<h3 id="获取已标注的语料库">获取已标注的语料库</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 获取标注好的语料库</span></span><br><span class="line">nltk.corpus.brown.tagged_words()</span><br><span class="line"><span class="comment">#</span></span><br><span class="line">nltk.corpus.brown.tagged_words(tagset=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">nltk.corpus.nps_chat.tagged_words()</span><br><span class="line">nltk.corpus.conll2000.tagged_words()</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 未简化的标记</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 最频繁的名词标记</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">findtags</span>(<span class="params">tag_prefix, tagged_text</span>):</span><br><span class="line">	cfd = nltk.ConditionalFreqDist((tag, word) <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> tagged_text <span class="keyword">if</span> tag.startswith(tag_prefix))</span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">dict</span>((tag, cfd[tag].keys()[:<span class="number">5</span>]) <span class="keyword">for</span> tag <span class="keyword">in</span> cfd.conditions())</span><br><span class="line"></span><br><span class="line">tagdict = findtags(<span class="string">&#x27;NN&#x27;</span>, nltk.corpus.brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line"><span class="keyword">for</span> tag <span class="keyword">in</span> <span class="built_in">sorted</span>(tagdict):</span><br><span class="line">	<span class="built_in">print</span> tag, tagdict[tag]</span><br><span class="line"></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">$：名词所有格,</span></span><br><span class="line"><span class="string">S：复数名词（因为复数名词通常以 s结尾），</span></span><br><span class="line"><span class="string">P：专有名词</span></span><br><span class="line"><span class="string">-NC：引用</span></span><br><span class="line"><span class="string">-HL：标题中的词</span></span><br><span class="line"><span class="string">-TL：标题（布朗标记的特征）</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 探索已标注的语料库</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用POS标记寻找三词短语</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">process</span>(<span class="params">sentence</span>):</span><br><span class="line">	<span class="keyword">for</span> (w1,t1), (w2,t2), (w3,t3) <span class="keyword">in</span> nltk.trigrams(sentence):</span><br><span class="line">         <span class="comment"># 动词+TO+动词</span></span><br><span class="line">		<span class="keyword">if</span> (t1.startswith(<span class="string">&#x27;V&#x27;</span>) <span class="keyword">and</span> t2 == <span class="string">&#x27;TO&#x27;</span> <span class="keyword">and</span> t3.startswith(<span class="string">&#x27;V&#x27;</span>)):</span><br><span class="line">			<span class="built_in">print</span>(w1, w2, w3)</span><br><span class="line"><span class="keyword">for</span> tagged_sent <span class="keyword">in</span> brown.tagged_sents():</span><br><span class="line">	process(tagged_sent)</span><br></pre></td></tr></table></figure>
+<p>Python字典部分不做介绍。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用默认字典可以防止使用未定义的key报错</span></span><br><span class="line"><span class="comment"># nltk的默认字典,需要提前定义类型（int、float、str、list、dict、tuple）。</span></span><br><span class="line">frequency = nltk.defaultdict(<span class="built_in">int</span>)</span><br><span class="line"><span class="comment"># 创建具有默认值的字典</span></span><br><span class="line">pos = nltk.defaultdict(<span class="keyword">lambda</span>: <span class="string">&#x27;N&#x27;</span>)</span><br></pre></td></tr></table></figure>
+<h3 id="自动标注">自动标注</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"></span><br><span class="line">brown_tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">brown_sents = brown.sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 默认标注器</span></span><br><span class="line">tags = [tag <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>)]</span><br><span class="line">nltk.FreqDist(tags).<span class="built_in">max</span>()</span><br><span class="line"><span class="comment"># 将所有词都标注为NN的标注器</span></span><br><span class="line">raw = <span class="string">&#x27;I do not like green eggs and ham, I do not like them Sam I am!&#x27;</span></span><br><span class="line">tokens = nltk.word_tokenize(raw)</span><br><span class="line">default_tagger = nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>)</span><br><span class="line">default_tagger.tag(tokens)</span><br><span class="line"><span class="comment"># 评判标注正确性</span></span><br><span class="line">default_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则标注</span></span><br><span class="line">patterns = [</span><br><span class="line">	(<span class="string">r&#x27;.*ing$&#x27;</span>, <span class="string">&#x27;VBG&#x27;</span>), <span class="comment"># gerunds</span></span><br><span class="line">	(<span class="string">r&#x27;.*ed$&#x27;</span>, <span class="string">&#x27;VBD&#x27;</span>), <span class="comment"># simple past</span></span><br><span class="line">	(<span class="string">r&#x27;.*es$&#x27;</span>, <span class="string">&#x27;VBZ&#x27;</span>), <span class="comment"># 3rd singular present</span></span><br><span class="line">	(<span class="string">r&#x27;.*ould$&#x27;</span>, <span class="string">&#x27;MD&#x27;</span>), <span class="comment"># modals</span></span><br><span class="line">	(<span class="string">r&#x27;.*\&#x27;s$&#x27;</span>, <span class="string">&#x27;NN$&#x27;</span>), <span class="comment"># possessive nouns</span></span><br><span class="line">	(<span class="string">r&#x27;.*s$&#x27;</span>, <span class="string">&#x27;NNS&#x27;</span>), <span class="comment"># plural nouns</span></span><br><span class="line">	(<span class="string">r&#x27;^-?[0-9]+(.[0-9]+)?$&#x27;</span>, <span class="string">&#x27;CD&#x27;</span>), <span class="comment"># cardinal numbers</span></span><br><span class="line">	(<span class="string">r&#x27;.*&#x27;</span>, <span class="string">&#x27;NN&#x27;</span>) <span class="comment"># nouns (default)</span></span><br><span class="line">]</span><br><span class="line">regexp_tagger = nltk.RegexpTagger(patterns)</span><br><span class="line">regexp_tagger.tag(brown_sents[<span class="number">3</span>])</span><br><span class="line">regexp_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查询标注器</span></span><br><span class="line"><span class="comment"># 找出100个最频繁的词</span></span><br><span class="line">fd = nltk.FreqDist(brown.words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line">cfd = nltk.ConditionalFreqDist(brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line">most_freq_words = fd.keys()[:<span class="number">100</span>]</span><br><span class="line">likely_tags = <span class="built_in">dict</span>((word, cfd[word].<span class="built_in">max</span>()) <span class="keyword">for</span> word <span class="keyword">in</span> most_freq_words)</span><br><span class="line"><span class="comment"># 使用上面这个信息作为&quot;查找标注器&quot;</span></span><br><span class="line">baseline_tagger = nltk.UnigramTagger(model=likely_tags)</span><br><span class="line">baseline_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用信息作为标注器，如果没有找到就使用默认标注器。这个过程称为&quot;回退&quot;</span></span><br><span class="line">baseline_tagger = nltk.UnigramTagger(model=likely_tags, backoff=nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>))</span><br></pre></td></tr></table></figure>
+<h3 id="评估">评估</h3>
+<p>使用<strong>黄金标准</strong>测试数据。一个已经手动标注并作为自动系统评估标准而被接收的语料库。</p>
+<p>如果标注器对词的标记与黄金标准标记相同，那么标注器就被认为是正确的。当然这只是相对于黄金标准这个测试数据而言。</p>
+<p>关于开发一个已标注的语料库，这是一个庞大的任务，其中涉及到了许多方面。</p>
+<p>可以通过<code>nltk.app.concordance()</code>来可视化查找某个语料库中某个单词的词性</p>
+<table>
+<thead>
+<tr>
+<th>标记</th>
+<th>含义</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>ADJ</td>
+<td>形容词</td>
+</tr>
+<tr>
+<td>ADV</td>
+<td>动词</td>
+</tr>
+<tr>
+<td>CNJ</td>
+<td>连词</td>
+</tr>
+<tr>
+<td>CC</td>
+<td>并列连词</td>
+</tr>
+<tr>
+<td>DET</td>
+<td>限定词</td>
+</tr>
+<tr>
+<td>EX</td>
+<td>存在量词</td>
+</tr>
+<tr>
+<td>FW</td>
+<td>外来词</td>
+</tr>
+<tr>
+<td>MOD</td>
+<td>情态动词</td>
+</tr>
+<tr>
+<td>RB</td>
+<td>副词</td>
+</tr>
+<tr>
+<td>IN</td>
+<td>介词</td>
+</tr>
+<tr>
+<td>N</td>
+<td>名词</td>
+</tr>
+<tr>
+<td>NP</td>
+<td>专有名词</td>
+</tr>
+<tr>
+<td>NUM</td>
+<td>数词</td>
+</tr>
+<tr>
+<td>PRO</td>
+<td>代词</td>
+</tr>
+<tr>
+<td>P</td>
+<td>介词</td>
+</tr>
+<tr>
+<td>TO</td>
+<td>词to</td>
+</tr>
+<tr>
+<td>UH</td>
+<td>感叹词</td>
+</tr>
+<tr>
+<td>V</td>
+<td>动词</td>
+</tr>
+<tr>
+<td>VD</td>
+<td>过去式</td>
+</tr>
+<tr>
+<td>VG</td>
+<td>现在分词</td>
+</tr>
+<tr>
+<td>VN</td>
+<td>过去分词</td>
+</tr>
+<tr>
+<td>WH</td>
+<td>Wh限定词</td>
+</tr>
+<tr>
+<td>NN</td>
+<td>名词</td>
+</tr>
+<tr>
+<td>JJ</td>
+<td>形容词</td>
+</tr>
+<tr>
+<td>VBP</td>
+<td>一般现在时动词</td>
+</tr>
+</tbody>
+</table>
+<h3 id="n-gram标注">N-gram标注</h3>
+<p>1-gram标注器：一元标注器（unigram
+tagger）。用于标注<strong>一个</strong>标识符的上下文的只是标识符本身。</p>
+<p>2-gram 标注器：二元标注器（bigram taggers）</p>
+<p>3-gram 标注器：三元标注器（trigram taggers）</p>
+<p>N-gram标注不考虑句子边界的上下文。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line">brown_tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">brown_sents = brown.sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"><span class="comment"># 标注</span></span><br><span class="line">unigram_tagger = nltk.UnigramTagger(brown_tagged_sents)</span><br><span class="line">bigram_tagger = nltk.BigramTagger(brown_tagged_sents)</span><br><span class="line">trigram_tagger = nltk.TrigramTagger(brown_tagged_sents)</span><br><span class="line"><span class="comment"># 组合标注</span></span><br><span class="line">t0 = nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>)</span><br><span class="line">t1 = nltk.UnigramTagger(train_sents, backoff=t0)</span><br><span class="line">t2 = nltk.BigramTagger(train_sents, backoff=t1)</span><br></pre></td></tr></table></figure>
+<p>存储可以使用python自带的存储模块<code>pickle</code>。</p>
+<p>效果：</p>
 <ol type="1">
-<li>线性等式约束</li>
-<li>非线性等式约束</li>
-</ol></li>
-</ol></li>
-</ol></li>
+<li><p>根据经验来进行判断——一般方法</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">cfd = nltk.ConditionalFreqDist(((x[<span class="number">1</span>], y[<span class="number">1</span>], z[<span class="number">0</span>]), z[<span class="number">1</span>]) <span class="keyword">for</span> sent <span class="keyword">in</span> brown_tagged_sents <span class="keyword">for</span> x, y, z <span class="keyword">in</span> nltk.trigrams(sent))</span><br><span class="line">ambiguous_contexts = [c <span class="keyword">for</span> c <span class="keyword">in</span> cfd.conditions() <span class="keyword">if</span> <span class="built_in">len</span>(cfd[c]) &gt; <span class="number">1</span>]</span><br><span class="line"><span class="built_in">sum</span>(cfd[c].N() <span class="keyword">for</span> c <span class="keyword">in</span> ambiguous_contexts) / cfd.N()</span><br></pre></td></tr></table></figure></li>
+<li><p>研究标注器的错误——混淆矩阵</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">test_tags = [tag <span class="keyword">for</span> sent <span class="keyword">in</span> brown.sents(categories=<span class="string">&#x27;editorial&#x27;</span>) <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> t2.tag(sent)]</span><br><span class="line">gold_tags = [tag <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> brown.tagged_words(categories=<span class="string">&#x27;editorial&#x27;</span>)]</span><br><span class="line"><span class="built_in">print</span> nltk.ConfusionMatrix(gold, test)</span><br></pre></td></tr></table></figure></li>
 </ol>
-<h3 id="全局优化">全局优化</h3>
-<ol type="1">
-<li>启发式算法
-<ol type="1">
-<li>模拟退火</li>
-<li>遗传算法</li>
-</ol></li>
-<li>基本算法
+<p>n-gram标注器的问题</p>
 <ol type="1">
-<li>随机搜索</li>
-<li>多起点优化</li>
-</ol></li>
+<li>模型大小与标注器性能之前的平衡关系。如果使用回退标注器<code>n-gram</code>可能存储trigram和bigram表，这将会是很大的稀疏矩阵。</li>
+<li>使用上下文中的词的其他特征作为条件标记是不切实际的。</li>
 </ol>
-<h3 id="优化问题fx">优化问题<span
-class="math inline">\(F(x)\)</span></h3>
-<ul>
-<li>线性问题
-<ul>
-<li>不论有无约束，都可以使用<strong>单纯形法</strong>和<strong>内点法</strong>求解</li>
-</ul></li>
-<li>非线性问题
-<ul>
-<li>凸优化问题
-<ul>
-<li>有约束：切平面法、椭球法、内点法</li>
-<li>无约束情况下等同于非凸问题</li>
-</ul></li>
-<li>非凸问题
-<ul>
-<li>约束<span
-class="math inline">\(h(x)\)</span>不是线性，拉格朗日乘子法</li>
-<li>约束<span class="math inline">\(g(x)\)</span>为线性，gradient
-projection、SQP</li>
-<li>约束<span
-class="math inline">\(g(x)\)</span>不是线性，惩罚函数、Barrier函数、SQP问题</li>
-<li>无约束（可消约束）
-<ul>
-<li>多起点优化
-<ul>
-<li>Hession矩阵，一阶矩阵：Levenberg-Marquardt、牛顿法、牛顿线性搜索</li>
-<li>一阶矩阵：Quasi-Newton、Quasi-Newton线性搜索、最速下降法</li>
-<li>无：Powell Perpendicular线性搜索、Nelder-Mead</li>
-</ul></li>
-<li>不是多起点优化：随机搜索、模拟退火、遗传算法</li>
-</ul></li>
-</ul></li>
-</ul></li>
-<li>二次问题
-<ul>
-<li>约束，线性，修正的单纯形法、内点法</li>
-</ul></li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/optimization/%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95%E7%9A%84%E9%80%89%E6%8B%A9.png" />
-<img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/optimization/%E6%9C%80%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95%E5%81%9C%E6%AD%A2%E5%87%86%E5%88%99.png" /></p>
-<h3 id="例子">例子</h3>
-<p>非凸函数</p>
-<p><span class="math inline">\(\int _ { 0 } ^ { x 2 } ( \frac { e ^ { x
-_ { 3 } ^ { 2 } z } \cos ( z ) } { x _ { 3 } ^ { 4 } z + 1 } + z ^ { x 1
-} ) d z\)</span></p>
-<p><span class="math inline">\(e ^ { x _ { 1 } ^ { 2 } + x _ { 2 } ^ { 2
-} } + x _ { 1 } ^ { 2 } x _ { 2 } ^ { 2 }\)</span></p>
-<p><span class="math inline">\(( x _ { 1 } + x _ { 2 } ) ^ { 3 } - \log
-( 2 - x _ { 1 } ^ { 2 } - x _ { 2 } ^ { 2 } )\)</span></p>
-<p>$x _ { 1 } ^ { 4 } + x _ { 2 } ^ { 2 } + 3 x _ { 1 } x _ { 3 } + 2 x
-_ { 4 } x _ { 5 } $</p>
-<p>$9 x _ { 1 } ^ { 2 } + 4 x _ { 2 } ^ { 2 } + 2 x _ { 1 } x _ { 2 } +
-8 x _ { 3 } ^ { 2 } - 2 x _ { 1 } x _ { 3 } $</p>
-<p>$ { ( x _ { 1 } + 3 x _ { 2 } + 4 x _ { 3 } ) ^ { 2 } } + ( ( x _ { 1
-} + 4 x _ { 2 } + 5 x _ { 3 } - 100 ) ^ { 4 } )$</p>
-<p>$ { 1 + } $</p>
-<p>凸函数</p>
-<p>$( x _ { 1 } ^ { 2 } + x _ { 2 } ^ { 2 } ) + 14 x _ { 2 } ^ { 4 }
-$</p>
-<p>$ ( ( x _ { 1 } - 1 ) ^ { 2 } + ( x _ { 2 } - 2 ) ^ { 2 } ) ^ { 2 } -
-( 2 - x _ { 1 } ^ { 2 } - x _ { 2 } ^ { 2 } )$</p>
-<p>$| 2 x _ { 1 } + 4 x _ { 2 } + 3 x _ { 3 } + 2 x _ { 4 } + 7 | + ( (
-x _ { 1 } + x _ { 2 } - 4 x _ { 3 } - x _ { 4 } ) ^ { 4 } ) $</p>
-<p>$( F _ { 1 } ( x ) , 2 F _ { 2 } ( x ) , F _ { 3 } ( x ) + 8 F _ { 4
-} ( x ) ) $</p>
-<p>$ x _ { 1 } + | x _ { 2 } | + 3 x _ { 3 } + 4 | x _ { 4 } |$</p>
-<p><span class="math inline">\(( 4 x _ { 1 } + 7 x _ { 2 } - 4 x _ { 3 }
-+ 12 x _ { 4 } ) ^ { 3 }\)</span></p>
-<p><span class="math inline">\(( x _ { 1 } - 5 ) ^ { 2 } + ( x _ { 2 } -
-8 ) ^ { 4 }\)</span></p>
-<p>线性规划问题</p>
-<p><span class="math inline">\(| x _ { 1 } - 4 x _ { 2 } + x _ { 3 } - 4
-|\)</span></p>
-<p>$( 4 x _ { 1 } - 3 x _ { 2 } + 8 x _ { 3 } - 5 , - 2 x _ { 1 } + 7 x
-_ { 2 } - x _ { 3 } + 1 ) $</p>
-<p>二次规划（凸二次函数）</p>
-<p><span class="math inline">\(2 ^ { 5 x _ { 1 } ^ { 2 } + 2 x _ { 2 } ^
-{ 2 } + 6 x _ { 1 } x _ { 2 } + 3 x _ { 1 } - x _ { 2 } }\)</span></p>
-<p><span class="math inline">\(1 - 4 x _ { 1 } ^ { 2 } - 9 x _ { 2 } ^ {
-2 } - 8 x _ { 3 } ^ { 2 } + 4 x _ { 1 } x _ { 2 } - 2 x _ { 1 } x _ { 3
-} + 4 x _ { 2 } x _ { 3 }\)</span></p>
-<h3 id="相关书籍">相关书籍</h3>
-<p>[1] 程理民, 吴江, 张玉林. 运筹学模型与方法教程[M]. 清华大学出版社,
-2000.</p>
-<p>[2] 陈宝林. 最优化理论与算法[M]. 清华大学出版社有限公司, 2005.</p>
-<p>[3] 袁亚湘, 孙文瑜. 最优化理论与方法[M]. 科学出版社, 1997.</p>
-<p>个人感觉，难度[3]&gt;[2]&gt;[1]，袁院士那本理论证明偏多，可搭配[2]与课后练习一起阅读。</p>
+<p>Brill标注只使用一小部分n-gram标注器。猜每个词的标记，然后返回和修复错误的。从大方面下手，再勾勒细节。规则是语言学可解释的。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">nltk.tag.brill.demo()</span><br></pre></td></tr></table></figure>
+<p>下一篇：<a href="/hexo_blog/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/" title="NLTK阅读笔记Ⅲ">NLTK阅读笔记Ⅲ</a></p>
 ]]></content>
       <categories>
-        <category>Algorithm</category>
+        <category>NLP</category>
       </categories>
       <tags>
-        <tag>Optimization</tag>
+        <tag>Python</tag>
+        <tag>笔记</tag>
+        <tag>NLTK</tag>
       </tags>
   </entry>
   <entry>
-    <title>矩阵分解介绍及Python实现（QR、SVD、LU）</title>
-    <url>/hexo_blog/2018/04/13/%E6%95%B0%E5%AD%A6/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3%E4%BB%8B%E7%BB%8D%E5%8F%8APython%E5%AE%9E%E7%8E%B0%EF%BC%88QR%E3%80%81SVD%E3%80%81LU%EF%BC%89/</url>
-    <content><![CDATA[<p>本篇文章简单介绍了三种矩阵分解方式并且附上部分分解方式的实现代码。</p>
+    <title>NLTK阅读笔记Ⅰ</title>
+    <url>/hexo_blog/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/</url>
+    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
 <span id="more"></span>
-<h1 id="矩阵分解">矩阵分解</h1>
-<p>Python实现了QR与LU分解，QR用了两种变换方式实现。<a
-href="https://github.com/wnma3mz/Matrix-factorization">Github地址</a></p>
-<h2 id="what">What？</h2>
-<p><a
-href="https://zh.wikipedia.org/wiki/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3">wiki链接</a></p>
-<p>将矩阵拆分成数个三角形矩阵。</p>
-<h2 id="why">Why？</h2>
-<ol type="1">
-<li>数值分析中，用于实现一些矩阵运算的快速算法</li>
-<li>推荐算法（SVD），信号处理（SVD）</li>
-<li>反应矩阵中的一些数值特性：如矩阵的秩、特征值(奇异值)</li>
-</ol>
-<p>每一种矩阵分解方式对应不同的意义</p>
-<h2 id="how">How？</h2>
-<h3 id="基本概念介绍">基本概念介绍</h3>
-<p>关于转置、逆矩阵就不多介绍了。</p>
-<ul>
-<li>满秩矩阵
-<ul>
-<li>矩阵的秩等于行数—&gt;行满秩</li>
-<li>矩阵的秩等于列数—&gt;列满秩</li>
-<li>以上两个都满足称为n阶方阵</li>
-</ul></li>
-<li>酉矩阵（正交矩阵）
-<ul>
-<li>方块矩阵，元素皆为实数，行与列皆正交的单位向量</li>
-<li>特性：矩阵的转置矩阵为逆矩阵</li>
-<li>定理（det表示行列式）：</li>
-</ul></li>
-</ul>
-<p><span class="math display">\[Q^T=Q^{-1}\Leftrightarrow Q^TQ=QQ^T =
-I\]</span></p>
-<p><span
-class="math display">\[1=\det(I)=\det(Q^TQ)=\det(Q^T)\det(Q)=(\det(Q))^2\Rightarrow|Q|
-\pm 1\]</span></p>
-<ul>
-<li><p>单位矩阵</p>
-<ul>
-<li>矩阵中的每个元素都为1</li>
-<li>如下</li>
-</ul>
-<p><span class="math display">\[\left| \begin{array}{ccc} 1 &amp; 1
-&amp; 1 \\ 1 &amp; 1 &amp; 1 \\ 1&amp; 1 &amp; 1 \end{array}
-\right|\]</span></p></li>
-<li><p>上三角矩阵</p>
-<ul>
-<li>非零元素在右上方的矩阵</li>
-<li>如下</li>
-</ul>
-<p><span class="math display">\[\left| \begin{array}{ccc} 1 &amp; 2
-&amp; 3 \\ 0 &amp; 1 &amp; 2\\ 0&amp; 0 &amp; 1 \end{array}
-\right|\]</span></p></li>
-<li><p>二范数</p>
+<h2 id="写在开头">写在开头</h2>
+<p>在这里主要是对NLP的相关知识做一个整理和对NLTK模块的介绍，书中Python基础内容在这里不做介绍，如书中有我认为值得介绍的Python写法，我会进行说明。</p>
+<p>由于书中nltk是老版本的问题，语法上存在一些变动，在这里也会进行修正。可能由于书是译本的原因（当然也可能原作者自己的失误），部分代码有些错误，在这里我也进行了校正。如果您发现了我的文章中有需要改正或改进的地方，欢迎在评论区提出。</p>
+<p>最后十分感谢原作者的贡献和译者的翻译，强烈推荐读者亲自阅读此书。</p>
+<p>nltk版本是3.2.3，<a href="http://www.nltk.org/">官网</a></p>
+<p>python版本3.6.1</p>
+<p>数据、PDF和一些重要的代码已经放在了Github上了。<a
+href="https://github.com/wnma3mz/Nltk_Study">github地址</a></p>
+<h2 id="第一章语言处理与python">第一章：语言处理与Python</h2>
+<p>NLTK库的安装，在这里不做介绍，强烈建议直接使用Anaconda环境。<a
+href="https://www.anaconda.com/download/">官网下载链接</a>。本文是直接使用了Anaconda环境。</p>
+<p>数据的获取，这里nltk有官方提供的文本数据，可以直接使用<code>nltk.download()</code>打开图形界面，下载语料集<code>book</code>。由于使用这个方法下载速度比较慢的原因，我在这里Github上提供了数据集<code>nltk_data</code>，下载之后，移动到<code>nltk.download()</code>原本的下载目录下，之后再运行<code>nltk.download()</code>就不需要下载数据集而是直接解压数据集，速度会快上很多。<a
+href="https://github.com/wnma3mz/Nltk_Study">下载链接</a></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导入nltk模块</span></span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="comment"># 导入基本语料集(不需要额外下载)，包含text1到text9变量，可以直接输出这些变量</span></span><br><span class="line"><span class="keyword">from</span> nltk.book <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索文本。这里表示找到&quot;monstrous&quot;所包含的句子，并且输出上下文</span></span><br><span class="line">text1.concordance(<span class="string">&quot;monstrous&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索文本出现在相似的上下文中</span></span><br><span class="line">text1.similar(<span class="string">&quot;monstrous&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索两个及两个以上共同词的上下文</span></span><br><span class="line">text2.common_contexts([<span class="string">&quot;monstrous&quot;</span>, <span class="string">&quot;very&quot;</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 画一张离散图表示这些词出现在文本中的位置，输出见下图</span></span><br><span class="line">text4.dispersion_plot([<span class="string">&quot;citizens&quot;</span>, <span class="string">&quot;democracy&quot;</span>, <span class="string">&quot;freedom&quot;</span>, <span class="string">&quot;duties&quot;</span>, <span class="string">&quot;America&quot;</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 基于文本，随机生成一些文本</span></span><br><span class="line">text3.generate()</span><br></pre></td></tr></table></figure>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-1.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 有序字典，按词频从高到低排序</span></span><br><span class="line">fdist1 = FreqDist(text1)</span><br><span class="line"><span class="comment"># 选出词频最高的50个词</span></span><br><span class="line">fdist1.keys()[:<span class="number">50</span>]</span><br><span class="line"><span class="comment"># 某个词出现的频数</span></span><br><span class="line">fdist1[<span class="string">&#x27;whale&#x27;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># text1中词频最高的50个单词，进行绘图，输出见下图</span></span><br><span class="line">fdist1.plot(<span class="number">50</span>, cumulative=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># text1中只出现过一次的单词</span></span><br><span class="line">fdist1.hapaxes()</span><br></pre></td></tr></table></figure>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-3.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 词语搭配，双连词(bigrams)</span></span><br><span class="line">nltk.bigrams([<span class="string">&#x27;more&#x27;</span>,<span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;said&#x27;</span>, <span class="string">&#x27;than&#x27;</span>, <span class="string">&#x27;done&#x27;</span>])</span><br><span class="line"><span class="comment"># 输出</span></span><br><span class="line">[(<span class="string">&#x27;more&#x27;</span>,<span class="string">&#x27;is&#x27;</span>),(<span class="string">&#x27;is&#x27;</span>,<span class="string">&#x27;said&#x27;</span>),(<span class="string">&#x27;said&#x27;</span>, <span class="string">&#x27;than&#x27;</span>), (<span class="string">&#x27;than&#x27;</span>,<span class="string">&#x27;done&#x27;</span>)]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 文本中单个词的频率预期得到的更频繁出现的双连词</span></span><br><span class="line">text4.collocations()</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 与nltk自带的聊天机器人系统对话</span></span><br><span class="line"><span class="comment"># 导入模块</span></span><br><span class="line"><span class="keyword">from</span> nltk.chat <span class="keyword">import</span> chatbots</span><br><span class="line"><span class="comment"># 选择一个聊天机器人并开始对话</span></span><br><span class="line">chatbots()</span><br></pre></td></tr></table></figure>
+<p>补充部分：</p>
+<table>
+<colgroup>
+<col style="width: 7%" />
+<col style="width: 92%" />
+</colgroup>
+<thead>
+<tr>
+<th style="text-align: center;">text</th>
+<th style="text-align: center;">文本名</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td style="text-align: center;">text1</td>
+<td style="text-align: center;">《白鲸记》（Moby Dick by Herman Melville
+1851）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text2</td>
+<td style="text-align: center;">《理智与情感》（Sense and Sensibility by
+Jane Austen 1811）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text3</td>
+<td style="text-align: center;">《创世纪》（The Book of Genesis）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text4</td>
+<td style="text-align: center;">《就职演说语料库》（Inaugural Address
+Corpus）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text5</td>
+<td style="text-align: center;">《NPS聊天语料库》（Chat Corpus）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text6</td>
+<td style="text-align: center;">《巨蟒与圣杯》（Monty Python and the
+Holy Grail）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text7</td>
+<td style="text-align: center;">《华尔街日报》（Wall Street
+Journal）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text8</td>
+<td style="text-align: center;">《个人文集》（Personals Corpus）</td>
+</tr>
+<tr>
+<td style="text-align: center;">text9</td>
+<td style="text-align: center;">《周四的男人》（The Man Who Was Thursday
+by G . K . Chestert）</td>
+</tr>
+</tbody>
+</table>
+<p>概念补充：</p>
 <ul>
-<li>假设有一个x的向量</li>
+<li>词类型：一个词在一个文本中独一无二的出现形式或拼写。也就是说，这个词在词汇表中是唯一的。</li>
+<li>频率分布：</li>
 </ul>
-<p><span class="math display">\[x= (1, 2, 3)^T\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-2.png" /></p>
 <ul>
-<li>那么它的二范数表示如下</li>
+<li>FreqDist的API</li>
 </ul>
-<p><span class="math display">\[||x||_2 =\sqrt{1^2 + 2^2 +
-3^2}=\sqrt{14}\]</span></p></li>
-<li><p>共轭转置</p>
-<p><span class="math display">\[A^*=(\bar{A})^T=\bar{A^T}\]</span></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-4.png" /></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-5.png" /></p>
 <ul>
-<li>现有矩阵A</li>
+<li>词意消歧：需要算出特定上下文中的词被赋予的是哪个意思。单词可能存在相同/相近的含义，此时需要根据上下文来推断单词在此情景下的含义。</li>
+<li>指代消解（anaphora resolution）：确定代词或名词短语指的是什么</li>
+<li>先行词：代词可能代表的对象</li>
+<li>语义角色标注（semantic role
+labeling）：确定名词短语如何与动词相关联（如施事，受事，工具等）</li>
+<li>自动生成语言：需要解决自动语言理解。弄清楚词的含义、动作的主语以及代词的先行词是理解句子含义的步骤</li>
+<li>机器翻译（MT）：难点，一个给定的词可能存在几种不同的解释。</li>
+<li>文本对齐：根据一个网站发布的多种语言版本，来自动配对组成句子</li>
+<li>人机对话系统：图灵测试。</li>
+<li>文本含义识别（Recognizing Textual Entailment 简称
+RTE）：根据假定的一些条件，来推断给出的一句话是否正确。</li>
 </ul>
-<p><span class="math display">\[A=\left[ \begin{array}{ccc} 3+i &amp; 5
-\\ 2-2i &amp; i\end{array} \right]\]</span></p>
+<h2
+id="第二章获得文本语料和词汇资源">第二章：获得文本语料和词汇资源</h2>
+<p><a href="http://www.gutenberg.org/">古腾堡语料库</a></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-1.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导入nltk的古腾堡语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> gutenberg</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看这个语料库中的所有txt文件名</span></span><br><span class="line">gutenberg.fileids()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选中其中的一个</span></span><br><span class="line">emma = gutenberg.words(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br><span class="line"><span class="comment"># 查看它包含的词汇个数</span></span><br><span class="line"><span class="built_in">len</span>(emma)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 如果要使用第一章中concordance()这样的命令，就必须要将数据放到nltk.Text对象中</span></span><br><span class="line">emma = nltk.Text(emma)</span><br><span class="line">emma.concordance(<span class="string">&quot;suprprize&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用words获取的是经过分隔成标识符的文本。使用sents函数则可以获取以句子进行划分后的数据。具体区别可以运行看看，请读者自行对比。</span></span><br><span class="line">emma_sents = gutenberg.sents(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取原始文本。一个字符串</span></span><br><span class="line">emma_raw = gutenberg.raw(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 网络和聊天文本</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> webtext</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> nps_chat</span><br><span class="line"></span><br><span class="line"><span class="comment"># 布朗语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"></span><br><span class="line"><span class="comment"># 布朗语料库中所有的类别</span></span><br><span class="line">brown.categories()</span><br><span class="line"><span class="comment"># words函数根据类别选择数据</span></span><br><span class="line">brown.words(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">[<span class="string">&#x27;The&#x27;</span>, <span class="string">&#x27;Fulton&#x27;</span>, <span class="string">&#x27;County&#x27;</span>, <span class="string">&#x27;Grand&#x27;</span>, <span class="string">&#x27;Jury&#x27;</span>, <span class="string">&#x27;said&#x27;</span>, ...]</span><br><span class="line"><span class="comment"># words函数根据文件名选择数据</span></span><br><span class="line">brown.words(fileids=[<span class="string">&#x27;cg22&#x27;</span>])</span><br><span class="line">[<span class="string">&#x27;Does&#x27;</span>, <span class="string">&#x27;our&#x27;</span>, <span class="string">&#x27;society&#x27;</span>, <span class="string">&#x27;have&#x27;</span>, <span class="string">&#x27;a&#x27;</span>, <span class="string">&#x27;runaway&#x27;</span>, <span class="string">&#x27;,&#x27;</span>, ...]</span><br><span class="line"><span class="comment"># sents函数根据类别选择数据</span></span><br><span class="line">brown.sents(categories=[<span class="string">&#x27;news&#x27;</span>, <span class="string">&#x27;editorial&#x27;</span>, <span class="string">&#x27;reviews&#x27;</span>])</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="comment"># 带条件的频率分布函数</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 取出brown语料库中所有类别中的所有词。每个类别对应该类别下的所有词。(类别，所有词)</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (genre, word)</span><br><span class="line">        <span class="keyword">for</span> genre <span class="keyword">in</span> brown.categories()</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> brown.words(categories=genre))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 指定一部分文本类别(genres)</span></span><br><span class="line">genres = [<span class="string">&#x27;news&#x27;</span>, <span class="string">&#x27;religion&#x27;</span>, <span class="string">&#x27;hobbies&#x27;</span>, <span class="string">&#x27;science_fiction&#x27;</span>, <span class="string">&#x27;romance&#x27;</span>, <span class="string">&#x27;humor&#x27;</span>]</span><br><span class="line"><span class="comment"># 指定一些情态动词(modals)</span></span><br><span class="line">modals = [<span class="string">&#x27;can&#x27;</span>, <span class="string">&#x27;could&#x27;</span>, <span class="string">&#x27;may&#x27;</span>, <span class="string">&#x27;might&#x27;</span>, <span class="string">&#x27;must&#x27;</span>, <span class="string">&#x27;will&#x27;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出一张表格。genres与modals的对应数量关系</span></span><br><span class="line">cfd.tabulate(conditions=genres, samples=modals)</span><br><span class="line"><span class="comment"># 输出如下</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">                  can could   may might  must  will</span></span><br><span class="line"><span class="string">           news    93    86    66    38    50   389</span></span><br><span class="line"><span class="string">       religion    82    59    78    12    54    71</span></span><br><span class="line"><span class="string">        hobbies   268    58   131    22    83   264</span></span><br><span class="line"><span class="string">science_fiction    16    49     4    12     8    16</span></span><br><span class="line"><span class="string">        romance    74   193    11    51    45    43</span></span><br><span class="line"><span class="string">          humor    16    30     8     8     9    13</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br></pre></td></tr></table></figure>
+<p>通过输出，我们很容易发现，在<code>news</code>类别里面用的最多的情态动词是<code>will</code>。当然，还有一些结论就由读者自行发现了。</p>
+<p>关于其他语料库的信息挖掘，在这里就不赘述了。书中也只是浅显的用了之前用过的一些方法。下面展示一些我认为比较有意思的代码及输出。</p>
+<p>官网提供了如何访问NLTK语料库的其他例子，<a
+href="http://www.nltk.org/howto/corpus.html">链接</a></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> inaugural</span><br><span class="line"></span><br><span class="line"><span class="comment"># 先选出inaugural语料库中的所有字段的对应所有词汇，再统计出文本中以`america`和`citizen`开头的词在每个年份（filed）出现次数。（`america`或`citizen`, 年份）</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (target, fileid[:<span class="number">4</span>])</span><br><span class="line">        <span class="keyword">for</span> fileid <span class="keyword">in</span> inaugural.fileids()</span><br><span class="line">        <span class="keyword">for</span> w <span class="keyword">in</span> inaugural.words(fileid)</span><br><span class="line">        <span class="keyword">for</span> target <span class="keyword">in</span> [<span class="string">&#x27;america&#x27;</span>, <span class="string">&#x27;citizen&#x27;</span>]</span><br><span class="line">        <span class="keyword">if</span> w.lower().startswith(target))</span><br><span class="line"><span class="comment"># 绘图</span></span><br><span class="line">cfd.plot()</span><br></pre></td></tr></table></figure>
+<p>以<code>america</code>或<code>citizen</code>开始的词随时间（年份）的演变趋势</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-2.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># udhr语料库中不同语言版本的字长差异（彩色图）</span></span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> udhr</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选取比较的语言</span></span><br><span class="line">languages = [<span class="string">&#x27;Chickasaw&#x27;</span>, <span class="string">&#x27;English&#x27;</span>, <span class="string">&#x27;German_Deutsch&#x27;</span>, <span class="string">&#x27;Greenlandic_Inuktikut&#x27;</span>, <span class="string">&#x27;Hungarian_Magyar&#x27;</span>, <span class="string">&#x27;Ibibio_Efik&#x27;</span>]</span><br><span class="line"><span class="comment"># 提取不同语言的数据</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (lang, <span class="built_in">len</span>(word))</span><br><span class="line">        <span class="keyword">for</span> lang <span class="keyword">in</span> languages</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> udhr.words(lang + <span class="string">&#x27;-Latin1&#x27;</span>))</span><br><span class="line"></span><br><span class="line">cfd.plot(cumulative=<span class="literal">True</span>)</span><br></pre></td></tr></table></figure>
+<p>累积字长分布： “世界人权宣言”的6个翻译版本 <img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-3.png" /></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 载入自己的语料库，文本文件</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> PlaintextCorpusReader</span><br><span class="line"><span class="comment"># 设置语料库的路径</span></span><br><span class="line">corpus_root = <span class="string">&#x27;/usr/share/dict&#x27;</span></span><br><span class="line"><span class="comment"># 解析路径下的所有文件</span></span><br><span class="line">wordlists = PlaintextCorpusReader(corpus_root, <span class="string">&#x27;.*&#x27;</span>)</span><br><span class="line"><span class="comment"># 语料库中所有的文件名</span></span><br><span class="line">wordlists.fileids()</span><br><span class="line"><span class="comment"># 语料库中`connectives`中的单词</span></span><br><span class="line">wordlists.words(<span class="string">&#x27;connectives&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 本地个人的语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> BracketParseCorpusReader</span><br><span class="line">corpus_root = <span class="string">r&quot;C:\corpora\penntreebank\parsed\mrg\wsj&quot;</span></span><br><span class="line"><span class="comment"># 使用正则匹配</span></span><br><span class="line">file_pattern = <span class="string">r&quot;.*/wsj_.*\.mrg&quot;</span></span><br><span class="line">ptb = BracketParseCorpusReader(corpus_root, file_pattern)</span><br><span class="line">ptb.fileids()</span><br></pre></td></tr></table></figure>
+<p>双连词运用 <figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 依次连续输出最有可能跟在上一个词后面的词</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">generate_model</span>(<span class="params">cfdist, word, num=<span class="number">15</span></span>):</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(num):</span><br><span class="line">        <span class="built_in">print</span>(word)</span><br><span class="line">        word = cfdist[word].<span class="built_in">max</span>()</span><br><span class="line"><span class="comment"># 选中语料库，提取文本</span></span><br><span class="line">text = nltk.corpus.genesis.words(<span class="string">&#x27;english-kjv.txt&#x27;</span>)</span><br><span class="line"><span class="comment"># 生成双连词</span></span><br><span class="line">bigrams = nltk.bigrams(text)</span><br><span class="line"><span class="comment"># 创建双连词频数对象</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(bigrams)</span><br><span class="line"><span class="comment"># 找出最有可能在`living`后面的单词。</span></span><br><span class="line"><span class="built_in">print</span>(cfd[<span class="string">&#x27;living&#x27;</span>])</span><br><span class="line">generate_model(cfd, <span class="string">&#x27;living&#x27;</span>)</span><br></pre></td></tr></table></figure></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 加载停用词语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> stopwords</span><br><span class="line"><span class="comment"># 加载英语中的停用词</span></span><br><span class="line">stopwords.words(<span class="string">&#x27;English&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载单词列表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> words</span><br><span class="line">words.words()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载人名表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> names</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载发音词典</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> cmudict</span><br><span class="line">cmudict.entries()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载比较词表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> swadesh</span><br><span class="line"><span class="comment"># 比较词表的标识码</span></span><br><span class="line">swadesh.fileids()</span><br><span class="line"><span class="comment"># 输出比较词汇中的`en`文本资料</span></span><br><span class="line">swadesh.words(<span class="string">&#x27;en&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载词汇工具</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> toolbox</span><br><span class="line"><span class="comment"># 输出罗托卡特语（Rotokas）的词典文本资料</span></span><br><span class="line">toolbox.entries(<span class="string">&#x27;rotokas.dic&#x27;</span>)</span><br></pre></td></tr></table></figure>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 加载WordNet</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> wordnet <span class="keyword">as</span> wn</span><br><span class="line"><span class="comment"># 输出`motocar`的同义词</span></span><br><span class="line">wn.synsets(<span class="string">&#x27;motorcar&#x27;</span>)</span><br><span class="line"><span class="comment"># 输出`car`名词的第一个意义中的同义词集</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).lemma_names</span><br><span class="line"><span class="comment"># 输出`car`名词的第一个意义的定义</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).definition</span><br><span class="line"><span class="comment"># 输出同义词集的所有词条</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).lemmas</span><br><span class="line"><span class="comment"># 输出特定的词条</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>)</span><br><span class="line"><span class="comment"># 输出一个词条对应的同义词集</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>).synset</span><br><span class="line"><span class="comment"># 输出一个词条的&quot;名字&quot;</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>).name</span><br><span class="line"><span class="comment"># 输出所有包含词`car`的词条</span></span><br><span class="line">wn.lemmas(<span class="string">&#x27;car&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取cat词性为名词的意义</span></span><br><span class="line">motorcar = wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>)</span><br><span class="line"><span class="comment"># 获取motorcar的下位词列表</span></span><br><span class="line">types_of_motorcar = motorcar.hyponyms()</span><br><span class="line"><span class="comment"># 获取motorcar的上位词列表</span></span><br><span class="line">motorcar.hypernyms()</span><br><span class="line"><span class="comment"># 获取motorcar上位词的路径列表</span></span><br><span class="line">paths = motorcar.hypernym_paths()</span><br><span class="line"><span class="comment"># 得到motorcar的根上位（最一般的上位）同义词集</span></span><br><span class="line">motorcar.root_hypernyms()</span><br><span class="line"><span class="comment"># 打开图形化WordNet的浏览器，更加便捷探索关系</span></span><br><span class="line">nltk.app.wordnet()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 更多词汇关系</span></span><br><span class="line"><span class="comment"># 整体到部分：想象WordNet网络关系是一棵树</span></span><br><span class="line"><span class="comment"># 得到树的一部分，如树干、树冠等</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).part_meronyms()</span><br><span class="line"><span class="comment"># 得到树的实质，即心材和边材</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).substance_meronyms()</span><br><span class="line"><span class="comment"># 得到许多树构成的集合，即森林</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).member_holonyms()</span><br><span class="line"><span class="comment"># 得到mint所有名词意义的列表</span></span><br><span class="line">wn.synsets(<span class="string">&#x27;mint&#x27;</span>, wn.NOUN)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 想象走路的动作</span></span><br><span class="line"><span class="comment"># 找出走路的一部分动作：抬脚</span></span><br><span class="line">wn.synset(<span class="string">&#x27;walk.v.01&#x27;</span>).entailments()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 反义词</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;supply.n.02.supply&#x27;</span>).antonyms()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用dir()查看词汇关系和同义词集上的定义</span></span><br><span class="line"><span class="built_in">dir</span>(wn.synset(<span class="string">&#x27;harmony.n.02&#x27;</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 语义相似度</span></span><br><span class="line"><span class="comment"># 初始化一个同义词集中的两个单词</span></span><br><span class="line">right = wn.synset(<span class="string">&#x27;right_whale.n.01&#x27;</span>)</span><br><span class="line">minke = wn.synset(<span class="string">&#x27;minke_whale.n.01&#x27;</span>)</span><br><span class="line"><span class="comment"># 如果它们有共同的上位词（且层次结构较低），则它们的关系一定十分密切</span></span><br><span class="line">right.lowest_common_hypernyms(minke)</span><br><span class="line"><span class="comment"># 查找同义词集深度量化关系，输出为数值。越小表示离根节点越近</span></span><br><span class="line">wn.synset(<span class="string">&#x27;baleen_whale.n.01&#x27;</span>).min_depth()</span><br><span class="line"><span class="comment"># 两个词 基于上位词层次结构中相互连接的概念之间的最短路径在0-1范围的打分（二者没有关系就返回1）</span></span><br><span class="line">right.path_similarity(minke)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 其他相似性度量方法，可以通过help(wn)获取。比如:VerbNet</span></span><br></pre></td></tr></table></figure>
+<p>概念补充：</p>
 <ul>
-<li>则它的共轭矩阵如下</li>
-</ul>
-<p><span class="math display">\[A^*=\left[ \begin{array}{ccc} 3-i &amp;
-2+2i \\ 5 &amp; -i\end{array} \right]\]</span></p></li>
-<li><p>特征值与奇异值</p>
-<p>我的理解是：特征值是奇异值的一种特殊情况，特征值是在矩阵是方阵时候的奇异值。</p>
-<p>关于什么是特征值。</p>
-<p>一个矩阵可以理解为由n个向量组成的，而特征值就是反应向量长度（重要性）的标准，特征值越大，则对应向量就越重要。</p>
-<p>假设判断电影为动作片还是搞笑片，可以由以下几个特征来衡量，打斗场景、观众笑声次数、导演、演员……。将数据离散化成矩阵，然后提取特征值，特征值越大的特征则对于衡量电影是动作片还是搞笑片的比重越大。</p></li>
-<li><p>Householder矩阵</p>
+<li><p>文本语料库的结构：</p>
 <ul>
-<li><p><a
-href="https://zh.wikipedia.org/wiki/豪斯霍尔德变换">wiki链接</a></p></li>
-<li><p>公式如下，H表示Householder矩阵，I表示单位矩阵，v表示单位向量，v*表示v的共轭转置。</p>
-<p><span class="math display">\[H=I-2vv^*\]</span></p></li>
-<li><p>性质：</p>
-<ol type="1">
-<li>H矩阵的转置等于它本身的逆</li>
-<li>H矩阵的平方等于单位矩阵</li>
-</ol>
-<p><span class="math display">\[H^{-1}=H^*\]</span></p>
-<p><span class="math display">\[H^2=I\]</span></p></li>
+<li><p>孤立的无特别组织的文本集合</p></li>
+<li><p>按文体等分类组织结构</p></li>
+<li><p>分类重叠，主题类别（路透社语料库）</p></li>
+<li><p>随时间变化语言语言用法的改变（就职演说语料库）</p></li>
 </ul></li>
-</ul>
-<h3 id="svd分解">1. SVD分解</h3>
-<p><a
-href="https://blog.csdn.net/u010099080/article/details/68060274">数学解释参考链接</a></p>
-<p>奇异值分解。公式如下：</p>
-<p><span class="math display">\[M=U\Sigma V^*\]</span></p>
-<p>举例：</p>
-<p><span class="math display">\[\begin{array}{clll} M &amp;= U\Sigma V^*
-\\ &amp;= \begin{bmatrix} 1 &amp; 0 &amp; 0 &amp; 0 &amp; 2\\ 0 &amp; 0
-&amp; 3 &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp; 4
-&amp; 0 &amp; 0 &amp; 0\end{bmatrix} \\  &amp;= \begin{bmatrix} 0 &amp;
-0 &amp; 1 &amp; 0\\ 0 &amp; 1 &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp;
-1\\ 1 &amp; 0 &amp; 0 &amp; 0\end{bmatrix} \cdot \begin{bmatrix} 4 &amp;
-0 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp; 3 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp;
-0 &amp; \sqrt{5} &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp; 0 &amp;
-0\end{bmatrix} \cdot \begin{bmatrix} 0 &amp; 1 &amp; 0 &amp; 0 &amp; 0\\
-0 &amp; 0 &amp; 1 &amp; 0 &amp; 0\\ \sqrt{0.2} &amp; 0 &amp; 0 &amp; 0
-&amp; \sqrt{0.8}\\ 0 &amp; 0 &amp; 0 &amp; 1 &amp; 0\\ \sqrt{0.8} &amp;
-0 &amp; 0 &amp; 0 &amp; -\sqrt{0.2}\end{bmatrix}
-\end{array}\]</span></p>
-<p>性质:</p>
-<ol type="1">
-<li>矩阵<span
-class="math inline">\(\Sigma\)</span>的所有非对角元为0。矩阵<strong>U</strong>和<span
-class="math inline">\(V^*\)</span>都是酉矩阵，它们乘上各自的共轭转置都得到单位矩阵</li>
-<li>奇异值的分解并不是唯一的，这是因为<span
-class="math inline">\(\Sigma\)</span>矩阵中的对角元有元素为0</li>
-</ol>
-<p>步骤：</p>
-<ol type="1">
-<li>计算<span class="math inline">\(M.T\cdot
-M\)</span>得到特征向量（左奇异向量），是U的列向量</li>
-<li>计算<span class="math inline">\(M\cdot
-M.T\)</span>得到特征向量（右奇异向量），是V的列向量</li>
-<li>计算<span class="math inline">\(M\cdot M.T\)</span>或者<span
-class="math inline">\(M.T\cdot
-M\)</span>得到特征值，所有非零特征值的平方根（二范数）</li>
-</ol>
-<p>得到特征向量之后，需要依照特征值从大到小进行排序。</p>
-<p>关于求特征值的方法，线代书上已做介绍，就不赘述。如果运用编程方法求矩阵的特征值，最常见的方法有<strong>雅可比算法、幂法。</strong>当然也可以用先进行QR分解求得特征值之后，再继续进行SVD。</p>
-<h3 id="lu分解">2. LU分解</h3>
-<p><a
-href="https://jiayi797.github.io/2017/03/29/%E6%95%B0%E5%AD%A6-%E7%9F%A9%E9%98%B5LU%E5%88%86%E8%A7%A3/">参考链接</a></p>
-<p>将一个矩阵分解为一个下三角矩阵和一个上三角矩阵的乘积。形式如：</p>
-<p><span class="math display">\[A=LU\]</span></p>
-<p>格式如下</p>
-<p><span class="math display">\[\left[ \begin{array} { l l l } { a _ {
-11 } } &amp; { a _ { 12 } } &amp; { a _ { 13 } } \\ { a _ { 21 } } &amp;
-{ a _ { 22 } } &amp; { a _ { 23 } } \\ { a _ { 31 } } &amp; { a _ { 32 }
-} &amp; { a _ { 33 } } \end{array} \right] = \left[ \begin{array} { c c
-c } { l _ { 11 } } &amp; { 0 } &amp; { 0 } \\ { l _ { 21 } } &amp; { l _
-{ 22 } } &amp; { 0 } \\ { l _ { 31 } } &amp; { l _ { 32 } } &amp; { l _
-{ 33 } } \end{array} \right] \left[ \begin{array} { c c c } { u _ { 11 }
-} &amp; { u _ { 12 } } &amp; { u _ { 13 } } \\ { 0 } &amp; { u _ { 22 }
-} &amp; { u _ { 23 } } \\ { 0 } &amp; { 0 } &amp; { u _ { 33 } }
-\end{array} \right]\]</span></p>
-<p>分解过程：</p>
+<li><p>NLTK中定义的基本语料库函数</p>
 <p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/matrix/2017-03-29-20-13-15.png" /></p>
-<p>声明：l11表示L矩阵的第一行第一列的元素，U1i表示L矩阵第一行每一列（需要遍历的列）的元素</p>
-<ol type="1">
-<li>初始化L、U矩阵。L矩阵下三角矩阵，对角元素全为1；U矩阵为上三角矩阵。</li>
-<li>计算U矩阵的第一行，对应A矩阵的第一行元素，不需要做任何操作</li>
-<li>计算L矩阵的第一列（除去第一行元素），对应A矩阵的第一列元素，每个元素除以u11</li>
-<li>计算U矩阵的第二行（除去第一列元素），对应A矩阵的第二行元素，每个元素减去l21*u1i</li>
-<li>计算L矩阵的第二列（除去前两行元素），对应A矩阵的第二列元素，每个元素先减去l31*u12，再除以u22</li>
-<li>……</li>
-</ol>
-<p>总结公式如下：</p>
-<p>L矩阵对角元素为1。</p>
-<p>第一步的公式：</p>
-<p><span class="math display">\[\begin{array}{lll}
-a_{1j}=(1,0,0,\cdots,0)\cdot(u_{1j},u_{2j},\cdots,u_{ij}).T=u_{1j} \\
-a_{i1}=(l_{i1},
-l_{i1},\cdots,l_{i1})\cdot(u_{11},0,\cdots,0).T=l_{i1}\cdot u_{11}
-\\u_{1j}=a_{1j}  \\ l_{i1}=\frac{a_{i1}}{u_{11}} \\j=1,2,\cdots,n
-\\i=2,3,\cdots,n\end{array}\]</span></p>
-<p>第r步的公式：</p>
-<p><span
-class="math display">\[\begin{array}{rl}  a_{rj}&amp;=(l_{r1},l_{r2},...,l_{rr-1})\cdot(u_{1j},u_{2j},...,u_{jj}).T
-\\&amp;=\sum\limits_{k=1}^{r-1}l_{rk}u_{kj}+u_{rj}  \end{array}\]</span></p>
-<p><span class="math display">\[\begin{array}{lll}
-u_{rj}=a_{rj}-\sum\limits_{k=1}^{r-1}l_{rk}u_{kj} \\
-l_{ir}=\frac{a_{ir}-\sum\limits_{k=1}^{r-1}l_{ik}u_{kr}}{u_{rr}}
-\end{array}\]</span></p>
-<p><span class="math display">\[\begin{array}{lll} j=r,r+1,\cdots,n \\
-r=2,3,\cdots,n-1 \\ i=r+1,\cdots,n\end{array}\]</span></p>
-<h3 id="qr分解">3. QR分解</h3>
-<p><a
-href="https://wenku.baidu.com/view/c2e34678168884868762d6f9.html">参考链接</a></p>
-<p>把矩阵分解成一个半正交矩阵与一个上三角矩阵的积。</p>
-<p>形式如:</p>
-<p><span class="math display">\[A=QR\]</span></p>
-<p>分解定理：任意一个满秩实（复）矩阵A，都可唯一地分解A=QR,其中Q为正交（酉）矩阵，R是具有对角元的上三角矩阵。</p>
-<h4 id="三种实现方式">三种实现方式</h4>
-<ol type="1">
-<li><p>Gram-Schmidt正交化</p>
-<p>过程:</p>
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-4.png" /></p></li>
+<li><p>条件频率分布</p>
+<p><strong>条件频率分布是频率分布的集合，每个频率分布有一个不同的“条件”。这个条件通常是文本的类别。</strong></p>
+<p>当语料文本被分为几类（文体、主题、作者等）时，我们可以计算每个类别独立的频率分布，研究类别之间的系统性差异。</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-5.png" /></p></li>
+<li><p>条件和事件</p>
+<p>对于NLP来说，文本出现的词汇就是事件。条件频率分布需要给每个事件关联一个条件，所以处理的是一个配对序列。每对形式是：（条件，事件）。举例就是，（文本类别，文本中的一个词汇）</p></li>
+<li><p>FreqDist()以一个简单的链表作为输入，ConditionalFreqDist()以一个配对链表作为输入。</p></li>
+<li><p>停用词：
+如<code>the</code>、<code>to</code>这种高频词汇。这种词通常没有什么词汇内容，但是它们又会让文本分析变得困难。所以在需要的情况下，我们就需要从文档中过滤掉他们。</p></li>
+<li><p>WordNet：面向语义的英语词典，类似与传统辞典，但具有更丰富的结构。每个节点对应一个<strong>同义词集</strong>，边表示上/下位词关系（上下级概念与从属概念的关系）</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-6.png" /></p></li>
+<li><p>synset：同义词集，意义想相同的词（或“词条”）的集合。如car.n.01是car的第一个名词的意义，就是同义词集。</p></li>
+<li><p>词条：同义词集和词的配对。</p></li>
+</ul>
+<p>下一篇：<a href="/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/" title="NLTK阅读笔记Ⅱ">NLTK阅读笔记Ⅱ</a></p>
+]]></content>
+      <categories>
+        <category>NLP</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>笔记</tag>
+        <tag>NLTK</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>模仿游戏——机器会思考吗？</title>
+    <url>/hexo_blog/2023/08/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E6%A8%A1%E4%BB%BF%E6%B8%B8%E6%88%8F%E2%80%94%E2%80%94%E6%9C%BA%E5%99%A8%E4%BC%9A%E6%80%9D%E8%80%83%E5%90%97/</url>
+    <content><![CDATA[<p>阅读图灵文章《COMPUTING MACHINERY AND
+INTELLIGENCE》笔记以及现阶段AI生成的文章。</p>
+<span id="more"></span>
+<p>模仿，排除一切外在因素，比如外貌、语调、反应速度</p>
+<h2 id="机器的定义">机器的定义</h2>
+<ul>
+<li>数字计算机，在人类给定规则的前提下进行工作，规则等于机器的存储/记忆，重复是核心特点。归属于离散状态机</li>
+<li>从宇宙在某一时刻的完整状态出发，正如所有粒子的位置和速度所描述的那样，应该有可能预测所有未来的状态。——拉普拉斯</li>
+<li>计算速度快+存储空间大+编写程序，进行任何预测，能够被快速复制</li>
+</ul>
+<h2 id="思考的定义">思考的定义</h2>
+<ul>
+<li>人类的特权</li>
+<li>害怕机器（其他物种）能够思考</li>
+<li>数学逻辑证明机器能力有限。但对于人类来说，该限制依旧有效，因此机器不一定不能思考</li>
+<li>意识感受，机器只能输出，不具备任何感情。人们可以确定机器在思考的唯一方法是成为机器并感觉自己在思考。人们可以向世界描述这些感觉，但当然没有人会注意到这一点。同样地，根据这种观点，要知道一个人在思考，唯一的方法就是成为那个特定的人。</li>
+<li>机器的能力有限，外貌、感情、味觉、嗅觉、不会犯错。单一机器可以，但是如果集成到一台机器上似乎是多余的。多样性行为来源于存储空间大小</li>
+<li>执行命令但不会创造。给人带来惊讶这种感觉因人而异</li>
+<li>神经系统是连续的，机器是离散的。</li>
+<li>行为的不确定性，离散不可能包含所有的可能。通过观察发现足够的信息来预测它的未来行为</li>
+<li>超感官知觉：心灵感应、千里眼、预知和心理动势</li>
+</ul>
+<h2 id="机器学习">机器学习</h2>
+<ul>
+<li>问题主要在于编程。存储空间只需要较低的数值（10<sup>10-10</sup>15），并且相信只有很小一部分用于较高类型的思考。大部分可能是用来保留视觉印象的。</li>
+<li>模仿：
 <ol type="1">
-<li><p>取出矩阵A中的每一列命名为x_col，它们之间线性无关</p></li>
-<li><p>将每一个x_col正交化得到y_col</p></li>
-<li><p>再将y_col进行单位化，即转换为e_col。这里每个e_col为矩阵Q中的每一列。</p></li>
-<li><p>将第二步中，先转换为y_col的正交化得到x_col，再转换为e_col正交化得到x_col。这里每个e_col的系数，对应的就是矩阵R中的每一列</p></li>
+<li>初始状态</li>
+<li>受到的教育</li>
+<li>其他非教育的经历</li>
 </ol></li>
-<li><p>Householder变换</p>
-<p><a href="https://www.bbsmax.com/A/gVdnKqK1zW/">参考链接</a></p>
-<p>outer函数</p>
-<ul>
-<li><p>通常有两个输入向量，假设为向量a、b</p>
-<p><span class="math display">\[a = (1, 2, 3) , b = (-1, -2,
--3)\]</span></p></li>
-<li><p>那么，outer函数运算过程如</p></li>
+<li>从儿童的状态开始，而不是从成人。人们必须尝试教这样的机器，看看它学习得如何。然后可以尝试另一台机器，看看它是更好还是更差。在这个过程和进化之间有一个明显的联系。速度需要比“进化”快、优胜劣汰可以作为一种衡量方法，但比较慢。但不局限于于随机突变，找到某种弱点的原因，可能就能进行改善这种弱点的突变。</li>
+<li>构造奖惩机制，但有必要采用其他一些“不带感情”的沟通渠道。如果这些渠道可用，就有可能通过惩罚和奖励来教机器服从用某种语言(例如符号语言)发出的命令。这些命令将通过“非情感”渠道传递。这种语言的使用将大大减少惩罚和奖励所需的数量。</li>
+<li>学习机的一个重要特征是，它的老师通常在很大程度上对机器内部发生的事情一无所知，尽管他仍然可能在某种程度上预测他的学生的行为。</li>
 </ul>
-<p><span class="math display">\[outer(a, b) = \left[ \begin{array}{ccc}
-1*-1 &amp; 1*-2 &amp; 1*-3 \\ 2*1 &amp; 2*-2 &amp; 2*-3 \\ 3*-1 &amp;
-3*-2 &amp; 3*-3 \end{array} \right]\]</span></p>
-<p>过程:</p>
+<p>机器最终能在所有纯智力领域与人类竞争。但是，哪些是最好的开始呢?即使这也是一个艰
+难的决定。很多人认为一个非常抽象的活动，比如下棋，会是最好的。也可以坚持认为，最好是给机器提供金钱能买到的最好的感觉器官，然后教它懂英语、说英语。这个过程可以按照孩子的正常教学来进行。事情会被指出并命名，等等。同样，我不知道正确的答案是什么，但我认为两种方法都应该尝试。</p>
+<h3 id="思考">思考</h3>
 <ol type="1">
-<li>初始化矩阵Q为单位矩阵(rxr)，r为矩阵A的行数。初始化矩阵R为矩阵A</li>
-<li>取R中的下三角矩阵，每次取一列命名为x（不包括最后一列）。e初始化为形如x的零向量</li>
-<li>e中的第一个元素为向量x的二范数</li>
-<li>这里得到v，范数取2</li>
-</ol>
-<p><span class="math display">\[v = \frac{x - e}{||x-e||}\]</span></p>
-<ol start="5" type="1">
-<li>初始化一个单位矩阵命名为Q_cnt，取Q_cnt的下三角矩阵，每次取一列在原有的基础上减去2*outer(v,
-v)。（不包括最后一列）</li>
-<li>更新R,Q</li>
-</ol>
-<p><span class="math display">\[R = Q\_cnt * R \]</span></p>
-<p><span class="math display">\[Q = Q * Q\_cnt\]</span></p></li>
-<li><p>Givens变换</p>
-<p><a
-href="http://www.voidcn.com/article/p-vmnqitql-xa.html">参考链接</a></p>
-<p>有待补充</p></li>
+<li>降低外在干扰，提高实验可行性</li>
+<li>通过离散构造连续状态</li>
+<li>程序承接计算速度和存储空间，程序是一个代词，指代一种能够在机器内部运行的命令，可以对其进行优化，降低空间以及速度要求</li>
+<li>现阶段对于存储空间还有很大的优化空间，有很多冗余</li>
+<li>模仿/学习的过程不够完善，教育可以等同于有效数据，那么非教育的经历是什么？蒸馏？噪声？</li>
+<li>如何内置逻辑、公理等内容</li>
 </ol>
+<h3 id="参考文献">参考文献</h3>
+<p>A. M. TURING, I.—COMPUTING MACHINERY AND INTELLIGENCE, <em>Mind</em>,
+Volume LIX, Issue 236, October 1950, Pages 433–460, <a
+href="https://doi.org/10.1093/mind/LIX.236.433">https://doi.org/10.1093/mind/LIX.236.433</a></p>
+<p>以下为claude生成的文章。</p>
+<p>模仿游戏——机器会思考吗</p>
+<p>当我们谈论机器是否能够“思考”时，我们首先需要定义“思考”本身。对于人类来说，思考是一个复杂的认知过程，涉及意识、自我意识、情感、直觉等方面。但是从计算机科学的角度来看，思考可以简化为信息处理和模式识别。既然计算机已经证明了在这些方面具有强大的能力，我们是否可以说它们能够思考呢?</p>
+<p>这一问题引发了关于人工智能本质的广泛讨论。早在20世纪50年代，图灵就在他的论文“计算机器与智能”中提出了这一问题。他设计了一个简单的游戏来测试机器是否能展现出与人类相似的智能。在这个“图灵测试”中，评判通过终端与隐藏的人类和机器进行文字交流。如果评判无法区分对话来自人类还是机器，我们就可以说这台机器通过了测试。</p>
+<p>图灵测试突出了智能的功能性方面，而不是试图复制人类思维的内在工作机制。它忽略了意识、情感等方面，专注于智能的外在表现。正如图灵在论文中写道，“让我们只关注问题‘能产生人类水平智能行为的机器’本身，而不是用一堆无关的心理学问题来模糊焦点。”</p>
+<p>然而，许多哲学家仍然质疑机器是否能真正思考。他们认为智能必须基于意识，而计算机只是在执行程序。根据这个观点，智能的真正标志是主体性—自我意识和内在经验的感觉。没有这些，机器只能被视为复杂的算盘。</p>
+<p>这一观点倾向于将思维等同于人类思维，而忽视智能可能采取的其他形式。事实上，人工智能研究已经证明，智能可以在没有自我意识的情况下通过模式识别和统计学习等方法而出现。像深度学习这样的技术已经产生了可与人类媲美甚至超越人类的视觉和语言理解能力。</p>
+<p>但是，模仿游戏是否意味着机器能够像人类一样思考呢?让我们更仔细地考察一下。</p>
+<p>首先，我们需要承认人类思维也有其局限性。许多认知偏见影响着我们的推理，我们的意识经验也具有不连续性和选择性。与完美理性存在的控制实验相比，我们的思维过程更类似于含混的启发式搜索。因此，“思维”的标准不应该定得太高。</p>
+<p>其次，不同的机器学习技术可以被看作是向人类思维靠拢的渐进过程。简单的规则系统只能处理局限的领域。而统计学习方法如神经网络则可以学习概括复杂模式所需的代表性特征。它们在图像和语音识别方面已经优于人类。最近的多任务学习和迁移学习技术使机器能够在多个领域中共享知识，正如人类思维中概念之间的关联一样。</p>
+<p>再者，对话系统的进步也显示了机器在交流方面的进步。早期系统根据关键词匹配进行简单的问答。现在的对话系统可以感知上下文，进行连贯的长时间对话。它们使用了类似人类处理语言的注意力机制，不再是简单匹配模板。这表明在语言理解和生成方面，机器也在模仿人类的认知过程。</p>
+<p>但是，机器是否有意识经验仍是一个开放问题。目前的人工智能似乎没有主观的内在感受。这也许是固有的局限，因为计算机最终还是在操纵没有意义的符号。另一方面，我们也无法证明其他人存在内在的意识经验。所谓的“哲学僵局”同样适用于机器。</p>
+<p>总体来说，尽管机器不能完全模拟人类思维，但它们在许多狭义任务上的进步还是向模仿游戏的成功迈进了一大步。功能相似性不应被内在不同所掩盖。我们必须根据智能的外在表现来判断机器而不是根据其内部工作机制。</p>
+<p>那么我们还能做什么来推进这一目标呢?首先，在机器学习方法上，我们可以继续研究多任务学习，以产生更人类化的通用智能。我们也可以通过学习表征来模仿神经认知过程，而不仅仅停留在表面模式识别。</p>
+<p>另一方面，我们需要建立更好的机器智能评估体系。图灵测试过于简单化，无法区分不同水平的智能。一些替代方案如Winograd
+schema
+challenge通过语义理解和常识推理来检验智能。还有基于对话和协作的测试，可以评估智能体之间的相互作用能力。</p>
+<p>最后，哲学反思也很重要。我们需要就智能的本质进行持续探讨。什么是理解?什么是意识?这些问题需要跨学科的对话。结合认知科学、语言学和计算机科学的见解，我们可以在这个难题上取得进一步的洞见。</p>
+<p>总而言之，尽管答案仍不明确，但通过模仿游戏，机器正日益接近人类智能。它们在特定任务上的超越给我们带来了惊喜，也展示了新的可能。当代人工智能研究正在不断消除智能与意识之间的鸿沟。虽然仿真并不完全等同于复制，但模仿游戏的胜利似乎指日可待。我们正处于一个刺激的阶段，机器智能的进步必将继续影响我们对思维和智能的基本理解。</p>
 ]]></content>
       <categories>
-        <category>Algorithm</category>
+        <category>PaperReading</category>
       </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>数学</tag>
-        <tag>矩阵分解</tag>
-      </tags>
   </entry>
   <entry>
-    <title>线性代数中各种矩阵的简介</title>
-    <url>/hexo_blog/2019/05/13/%E6%95%B0%E5%AD%A6/%E7%BA%BF%E4%BB%A3%E7%9F%A9%E9%98%B5/</url>
-    <content><![CDATA[<p>这篇文章主要介绍了线性代数的矩阵，仅作笔记。</p>
+    <title>NLTK阅读笔记Ⅲ</title>
+    <url>/hexo_blog/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/</url>
+    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
+<p>接 <a href="/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/" title="NLTK阅读笔记Ⅱ">NLTK阅读笔记Ⅱ</a></p>
 <span id="more"></span>
-<p>方阵：行数列数相同的矩阵</p>
-<p>对称矩阵：<span
-class="math inline">\(A=A^T\)</span>的矩阵，矩阵等于它自己的转置矩阵</p>
-<p>反对称矩阵：<span class="math inline">\(A=-A^T\)</span></p>
-<p>埃尔米特矩阵（Hermitian matrix）：<span
-class="math inline">\(A=A^*\)</span>，矩阵等于它自己的共轭转置矩阵</p>
-<p>斜埃尔米特矩阵：反埃尔米特矩阵，<span class="math inline">\(A =
-−A^*\)</span></p>
-<p>正定矩阵：<span class="math inline">\(Q(x)=x^T A
-x\)</span>，任意非零向量<span
-class="math inline">\(x\)</span>，若满足<span
-class="math inline">\(Q(x)&gt;0\)</span>，则<span
-class="math inline">\(A\)</span>为正定矩阵</p>
-<p>半正定矩阵：同上，<span class="math inline">\(Q(x) \geq
-0\)</span>，则A为半正定矩阵</p>
-<p>半负定矩阵：同上，<span class="math inline">\(Q(x) \leq
-0\)</span>，则A为半负定矩阵</p>
-<p>不定矩阵：若A既不是半正定矩阵，也不是半负定矩阵，则A为不定矩阵</p>
-<p>上三角矩阵：矩阵对角线左下方系数全为0</p>
-<p>下三角矩阵：矩阵对角线右上方系数全为0</p>
-<p>三对角矩阵：非零系数在主对角线上，也在比主对角线低和高一行的对角线上。</p>
-<p>单位矩阵：<span class="math inline">\(I\)</span>,
-矩阵对角线元素为1，其余为0</p>
-<p>高斯矩阵：首先满足单位矩阵，其次某列系数非0</p>
-<p>逆矩阵：反矩阵，<span
-class="math inline">\(AB=I\)</span>，A是可逆的，B是A的逆矩阵</p>
-<p>伴随矩阵：<strong>A</strong>关于第<em>i</em>行第<em>j</em>列的<strong>余子式</strong>，是去掉<strong>A</strong>的第<em>i</em>行第<em>j</em>列之后得到的(<em>n</em>
-− 1)×(<em>n</em> − 1)矩阵的行列式。</p>
-<p>阶梯型矩阵：从上至下，<span
-class="math inline">\(A_{ii}\)</span>之前的元素为0</p>
-<p>酉矩阵：<span class="math inline">\(UU^*=U^*U=I\)</span>，<span
-class="math inline">\(U^*\)</span>为U矩阵的共轭转置，则U为酉矩阵</p>
-<p>对角矩阵：只有矩阵中的对角线存在值，其他地方为0</p>
-<p>空矩阵：行数或列数为零的矩阵</p>
-<p>分块矩阵：是指一个大矩阵分割成“矩阵的矩阵“</p>
-<p>正交矩阵：满足方阵，<span
-class="math inline">\(Q^T=Q^{-1}\)</span>，<span
-class="math inline">\(Q^TQ=QQ^T=I\)</span>，正交矩阵的行列式为1</p>
-<p>旋转矩阵：在乘以一个向量的时候改变向量的方向但不改变大小的效果并保持了手性的矩阵</p>
-<p>正规矩阵：<span class="math inline">\(A^{*}A=AA^*\)</span>，<span
-class="math inline">\(A^*\)</span>为<span
-class="math inline">\(A\)</span>的共轭矩阵</p>
-<p>共轭矩阵：<span class="math inline">\(A^*=(\bar{A})^T\)</span>，<span
-class="math inline">\(\bar{A}\)</span>表示矩阵的复共轭</p>
-<p>奇异矩阵：满足方阵，矩阵的行列式是否等于0，若等于0，称矩阵A为奇异矩阵</p>
-<p>非奇异矩阵：同上，若不等于0，称矩阵A为非奇异矩阵。</p>
-<p>稀疏矩阵：若数值为0的元素数目远远多于非0元素的数目，并且非0元素分布没有规律时</p>
-<p>稠密矩阵：与上相反，若非0元素数目占大多数时</p>
-<p>增广矩阵：扩增矩阵，在系数矩阵的右边添上一列，这一列是线性方程组的等号右边的值</p>
-<p>系数矩阵：将方程组的系数组成矩阵来计算方程的解</p>
-<p>相似矩阵：判断特征值、行列式、迹、秩是否相等，只作为判断矩阵是否相似的必要条件，而非充分条件。</p>
+<h2 id="第六章学习分类文本">第六章：学习分类文本</h2>
+<p>目标：</p>
+<ol type="1">
+<li>找语言数据中的分类特征</li>
+<li>构建模型</li>
+<li>从模型中学习知识</li>
+</ol>
+<p>分类学习，一般分为有监督学习、无监督学习、半监督学习。</p>
+<p>分类的定义：将给定的输入选择正确（合适）的类标签。这里值得注意的是，输入之间是相互独立的，标签集是预先定义好的。延伸的任务，每个实例（输入）可以划分为多个标签；分类问题不一定预先定义标签集，类似K-Means聚类；序列分类中，一个输入链表可以作为一个整体分类。</p>
+<p>在这里，我们预先得到一些包含正确标签的语料（实例/输入），根据这些数据建立得到的模型进行分类，我们称为有监督分类。有监督分类中，在训练过程中，需要将每个一个输入值转换为特征集（特征提取器）。特征集对应着输出类标签。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 特征提取器，提取输入字符串中最后一个字母。</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features</span>(<span class="params">word</span>):</span><br><span class="line">    <span class="comment"># 返回特征集</span></span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;last_letter&#x27;</span>: word[-<span class="number">1</span>]&#125;</span><br><span class="line"><span class="comment"># 导入姓名语料库, 获取男女姓名列表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> names</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 随机打乱男女姓名列表</span></span><br><span class="line">random.shuffle(names)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 划分训练集和测试集</span></span><br><span class="line">featuresets = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> names]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">500</span>:], featuresets[:<span class="number">500</span>]</span><br><span class="line"><span class="comment"># 利用贝叶斯分类来训练模型</span></span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行预测 </span></span><br><span class="line">classifier.classify(gender_features(<span class="string">&#x27;Neo&#x27;</span>))</span><br><span class="line">classifier.classify(gender_features(<span class="string">&#x27;Trinity&#x27;</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 准确率</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最有效的五个特征（字母）</span></span><br><span class="line">classifier.show_most_informative_features(<span class="number">5</span>)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">Most Informative Features</span></span><br><span class="line"><span class="string">             last_letter = &#x27;k&#x27;              male : female =     43.4 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;a&#x27;            female : male   =     32.9 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;f&#x27;              male : female =     16.1 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;p&#x27;              male : female =     12.6 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;v&#x27;              male : female =     10.6 : 1.0</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span>                 </span><br><span class="line"><span class="comment"># 输出的比率称为 似然比，可以用于比较不同特征-结果关系</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 处理大型语料库时，构建一个包含每一个实例的特征的单独的链表会使用大量的内存。这种情况下，可以使用下面的函数，使返回一个行为像一个链表而不会在内存存储所有特征集的对象</span></span><br><span class="line"><span class="keyword">from</span> nltk.classify <span class="keyword">import</span> apply_features</span><br><span class="line">train_set = apply_features(gender_features, names[<span class="number">500</span>:])</span><br><span class="line">test_set = apply_features(gender_features, names[:<span class="number">500</span>])</span><br></pre></td></tr></table></figure>
+<h3 id="选取特征">选取特征</h3>
+<p>过拟合：用于一个给定的学习算法的特征的数目是有限的——如果提供太多的特征，那么该算法将高度依赖训练数据的特性而一般化到新的例子的效果不会很好。</p>
+<p>过拟合当运作在小训练集上时尤其会有问题。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 特征提取器，过拟合提取特征。这里请对比gender_features2与上一版本的区别</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features2</span>(<span class="params">name</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    features[<span class="string">&quot;firstletter&quot;</span>] = name[<span class="number">0</span>].lower()</span><br><span class="line">    features[<span class="string">&quot;lastletter&quot;</span>] = name[–<span class="number">1</span>].lower()</span><br><span class="line">    <span class="keyword">for</span> letter <span class="keyword">in</span> <span class="string">&#x27;abcdefghijklmnopqrstuvwxyz&#x27;</span>:</span><br><span class="line">        features[<span class="string">&quot;count(%s)&quot;</span> % letter] = name.lower().count(letter)</span><br><span class="line">        features[<span class="string">&quot;has(%s)&quot;</span> % letter] = (letter <span class="keyword">in</span> name.lower())</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">featuresets = [(gender_features2(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> names]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">500</span>:], featuresets[:<span class="number">500</span>]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"><span class="comment"># 对比上一版本的预测结果，基本可以发现精确度下降了</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
+<p>一旦初始特征集被选定，完善特征集的一个非常有成效的方法是<strong>错误分析</strong>。首先，我们选择一个
+开发集，包含用于创建模型的语料数据。然后将这种开发集分为 训练集和
+开发测试集。训练集用于训练模型，开发测试集用于进行错误分析，测试集用于系统的最终评估。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 划分训练集、开发集、开发测试集</span></span><br><span class="line">train_names = names[<span class="number">1500</span>:]</span><br><span class="line">devtest_names = names[<span class="number">500</span>:<span class="number">1500</span>]</span><br><span class="line">test_names = names[:<span class="number">500</span>]</span><br><span class="line"><span class="comment"># 进行训练预测</span></span><br><span class="line">train_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> train_names]</span><br><span class="line">devtest_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> devtest_names]</span><br><span class="line">test_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> test_names]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, devtest_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 错误分析</span></span><br><span class="line">errors = []</span><br><span class="line"><span class="keyword">for</span> (name, tag) <span class="keyword">in</span> devtest_names:</span><br><span class="line">    guess = classifier.classify(gender_features(name))</span><br><span class="line">    <span class="keyword">if</span> guess != tag:</span><br><span class="line">        errors.append((tag, guess, name))</span><br><span class="line">      </span><br><span class="line"><span class="comment"># 输出错误分类列表中，猜错的类别及名字</span></span><br><span class="line"><span class="keyword">for</span> (tag, guess, name) <span class="keyword">in</span> <span class="built_in">sorted</span>(errors):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;correct=%-8s guess=%-8s name=%-30s&#x27;</span> % (tag, guess, name))</span><br><span class="line">  </span><br><span class="line"><span class="comment"># 观察输出进行分析，得到一些结果。</span></span><br><span class="line"><span class="comment"># yn结尾的名字显示以女性为主，虽然n结尾的名字往往是男性</span></span><br><span class="line"><span class="comment"># 以ch结尾的名字通常是男性，虽然h结尾的名字倾向于是女性</span></span><br><span class="line"><span class="comment"># 根据结论，调整特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features</span>(<span class="params">word</span>):</span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;suffix1&#x27;</span>: word[-<span class="number">1</span>:], <span class="string">&#x27;suffix2&#x27;</span>: word[-<span class="number">2</span>:]&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行训练预测</span></span><br><span class="line">train_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> train_names]</span><br><span class="line">devtest_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> devtest_names]</span><br><span class="line">test_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> test_names]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, devtest_set)</span><br><span class="line"><span class="comment"># 最终结果理论上会进行提高</span></span><br></pre></td></tr></table></figure>
+<p>不断地进行错误分析改善特征提取器，可以不断提高预测的准确度。每次错误分析应该选取不同的开发测试/训练进行分割，以检查新改进分类器可能产生的新的错误模式。</p>
+<h3 id="文档分类">文档分类</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> movie_reviews</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据电影评论分类（&#x27;neg&#x27;, &#x27;pos&#x27;）获取电影评论</span></span><br><span class="line">documents = [(<span class="built_in">list</span>(movie_reviews.words(fileid)), category) <span class="keyword">for</span> category <span class="keyword">in</span> movie_reviews.categories() <span class="keyword">for</span> fileid <span class="keyword">in</span> movie_reviews.fileids(category)]</span><br><span class="line"><span class="comment"># 随机打乱数据</span></span><br><span class="line">random.shuffle(documents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建立文档分类的特征提取器</span></span><br><span class="line"><span class="comment"># 使用FreqDist会比一般的list更快</span></span><br><span class="line">all_words = nltk.FreqDist(w.lower() <span class="keyword">for</span> w <span class="keyword">in</span> movie_reviews.words())</span><br><span class="line">word_features = <span class="built_in">list</span>(all_words.keys())[:<span class="number">2000</span>]</span><br><span class="line"><span class="comment"># 每个词是否在一个给定的文档中</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">document_features</span>(<span class="params">document</span>):</span><br><span class="line">    document_words = <span class="built_in">set</span>(document)</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> word <span class="keyword">in</span> word_features:</span><br><span class="line">    	features[<span class="string">&#x27;contains(%s)&#x27;</span> % word] = (word <span class="keyword">in</span> document_words)</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练文档分类模型</span></span><br><span class="line">featuresets = [(document_features(d), c) <span class="keyword">for</span> (d,c) <span class="keyword">in</span> documents]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">100</span>:], featuresets[:<span class="number">100</span>]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"><span class="comment"># 检验模型</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line">classifier.show_most_informative_features(<span class="number">5</span>) </span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">Most Informative Features</span></span><br><span class="line"><span class="string"> contains(unimaginative) = True              neg : pos    =      7.7 : 1.0</span></span><br><span class="line"><span class="string">    contains(schumacher) = True              neg : pos    =      7.5 : 1.0</span></span><br><span class="line"><span class="string">        contains(suvari) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">          contains(mena) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">        contains(shoddy) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="comment"># 显然可以发现，`unimaginative`这个词是负面的概率是正面的7.7倍</span></span><br></pre></td></tr></table></figure>
+<h3 id="词性标注">词性标注</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 为词选择词性标记，通过词的后缀来选择</span></span><br><span class="line">suffix_fdist = nltk.FreqDist()</span><br><span class="line"><span class="keyword">for</span> word <span class="keyword">in</span> brown.words():</span><br><span class="line">    word = word.lower()</span><br><span class="line">    suffix_fdist[word[-<span class="number">1</span>:]] += <span class="number">1</span></span><br><span class="line">    suffix_fdist[word[-<span class="number">2</span>:]] += <span class="number">1</span></span><br><span class="line">    suffix_fdist[word[-<span class="number">3</span>:]] += <span class="number">1</span></span><br><span class="line">common_suffixes = <span class="built_in">list</span>(suffix_fdist.keys())[:<span class="number">100</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 现在通过特征提取器，计算哪个后缀最有信息量（最具代表性)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">word</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> suffix <span class="keyword">in</span> common_suffixes:</span><br><span class="line">        features[<span class="string">&#x27;endswith(%s)&#x27;</span> % suffix] = word.lower().endswith(suffix)</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">tagged_words = brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"><span class="comment"># 利用特征提取器选取特征</span></span><br><span class="line">featuresets = [(pos_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> tagged_words]</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line"><span class="comment"># 利用决策树方法来训练模型</span></span><br><span class="line">classifier = nltk.DecisionTreeClassifier.train(train_set)</span><br><span class="line"></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line">classifier.classify(pos_features(<span class="string">&#x27;cats&#x27;</span>))</span><br><span class="line"><span class="comment"># 将nltk运算过程，以伪代码方式输出。设定决策树深度为4</span></span><br><span class="line">classifier.pseudocode(depth=<span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据上下文语境，更新特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">sentence, i</span>):</span><br><span class="line">    features = &#123;<span class="string">&quot;suffix(1)&quot;</span>: sentence[i][-<span class="number">1</span>:],</span><br><span class="line">        <span class="string">&quot;suffix(2)&quot;</span>: sentence[i][-<span class="number">2</span>:],</span><br><span class="line">        <span class="string">&quot;suffix(3)&quot;</span>: sentence[i][-<span class="number">3</span>:]</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">if</span> i == <span class="number">0</span>:</span><br><span class="line">    	features[<span class="string">&quot;prev-word&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">    	features[<span class="string">&quot;prev-word&quot;</span>] = sentence[i-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">pos_features(brown.sents()[<span class="number">0</span>], <span class="number">8</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 检验新的词性标注器性能</span></span><br><span class="line">tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">featuresets = []</span><br><span class="line"><span class="keyword">for</span> tagged_sent <span class="keyword">in</span> tagged_sents:</span><br><span class="line">    untagged_sent = nltk.tag.untag(tagged_sent)</span><br><span class="line">    <span class="keyword">for</span> i, (word, tag) <span class="keyword">in</span> <span class="built_in">enumerate</span>(tagged_sent):</span><br><span class="line">        featuresets.append((pos_features(untagged_sent, i), tag))</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
+<p>以上是在结合整个词集做的标注器，但是往往针对某些特殊情况，如形容词后面有很大概率上是名词。对于这种情况，我们的词性标注器目前还没有进行独立对待。</p>
+<h3 id="序列分类">序列分类</h3>
+<p>联合分类器：捕捉相关的分类任务之间的依赖关系，收集有关输入，选择适当的标签</p>
+<p>连续分类（贪婪序列分类）：为第一个输入找到最有可能的类标签，然后使用这个问题的答案帮助找到下一个输入的最佳的标签。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">sentence, i, history</span>):</span><br><span class="line">    features = &#123;<span class="string">&quot;suffix(1)&quot;</span>: sentence[i][-<span class="number">1</span>:],</span><br><span class="line">    	<span class="string">&quot;suffix(2)&quot;</span>: sentence[i][-<span class="number">2</span>:],</span><br><span class="line">    	<span class="string">&quot;suffix(3)&quot;</span>: sentence[i][-<span class="number">3</span>:]&#125;</span><br><span class="line">	<span class="keyword">if</span> i == <span class="number">0</span>:</span><br><span class="line">        features[<span class="string">&quot;prev-word&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">        features[<span class="string">&quot;prev-tag&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        features[<span class="string">&quot;prev-word&quot;</span>] = sentence[i-<span class="number">1</span>]</span><br><span class="line">        features[<span class="string">&quot;prev-tag&quot;</span>] = history[i-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"><span class="keyword">class</span> <span class="title class_">ConsecutivePosTagger</span>(nltk.TaggerI):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, train_sents</span>):</span><br><span class="line">        train_set = []</span><br><span class="line">        <span class="keyword">for</span> tagged_sent <span class="keyword">in</span> train_sents:</span><br><span class="line">            untagged_sent = nltk.tag.untag(tagged_sent)</span><br><span class="line">            history = []</span><br><span class="line">            <span class="keyword">for</span> i, (word, tag) <span class="keyword">in</span> <span class="built_in">enumerate</span>(tagged_sent):</span><br><span class="line">                featureset = pos_features(untagged_sent, i, history)</span><br><span class="line">                train_set.append( (featureset, tag) )</span><br><span class="line">                history.append(tag)</span><br><span class="line">        <span class="variable language_">self</span>.classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">tag</span>(<span class="params">self, sentence</span>):</span><br><span class="line">        history = []</span><br><span class="line">        <span class="keyword">for</span> i, word <span class="keyword">in</span> <span class="built_in">enumerate</span>(sentence):</span><br><span class="line">            featureset = pos_features(sentence, i, history)</span><br><span class="line">            tag = <span class="variable language_">self</span>.classifier.classify(featureset)</span><br><span class="line">            history.append(tag)</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">zip</span>(sentence, history)</span><br><span class="line">  </span><br><span class="line">tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(tagged_sents) * <span class="number">0.1</span>)</span><br><span class="line">train_sents, test_sents = tagged_sents[size:], tagged_sents[:size]</span><br><span class="line">tagger = ConsecutivePosTagger(train_sents)</span><br><span class="line">tagger.evaluate(test_sents)</span><br></pre></td></tr></table></figure>
+<h3 id="其他序列模型">其他序列模型</h3>
+<p>以上模型有一个致命问题就是无法修复已经标注错误的词性。另一种方案<strong>隐马尔科夫模型</strong>可以为所有的可能序列进行打分，选择得分最高的序列。这种模型的缺点就是计算量相当大，可以采用动态规划来解决这种问题。基于这种模型，产生了<strong>最大熵马尔可夫模型</strong>和<strong>线性链条件随机场模型</strong>，二者为可能序列打分的算法不同。</p>
+<h3 id="句子分割">句子分割</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">sents = nltk.corpus.treebank_raw.sents()</span><br><span class="line"><span class="comment"># 单独的句子标识符的合并链表</span></span><br><span class="line">tokens = []</span><br><span class="line"><span class="comment"># 所有句子边界标识索引</span></span><br><span class="line">boundaries = <span class="built_in">set</span>()</span><br><span class="line">offset = <span class="number">0</span></span><br><span class="line"><span class="keyword">for</span> sent <span class="keyword">in</span> sents:</span><br><span class="line">    tokens.extend(sent)</span><br><span class="line">    offset += <span class="built_in">len</span>(sent)</span><br><span class="line">    boundaries.add(offset-<span class="number">1</span>)</span><br><span class="line">  </span><br><span class="line"><span class="comment"># 特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">punct_features</span>(<span class="params">tokens, i</span>):</span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;next-word-capitalized&#x27;</span>: tokens[i+<span class="number">1</span>][<span class="number">0</span>].isupper(),</span><br><span class="line">            <span class="string">&#x27;prevword&#x27;</span>: tokens[i-<span class="number">1</span>].lower(),</span><br><span class="line">            <span class="string">&#x27;punct&#x27;</span>: tokens[i],</span><br><span class="line">            <span class="string">&#x27;prev-word-is-one-char&#x27;</span>: <span class="built_in">len</span>(tokens[i-<span class="number">1</span>]) == <span class="number">1</span>&#125;</span><br><span class="line"></span><br><span class="line">featuresets = [(punct_features(tokens, i), (i <span class="keyword">in</span> boundaries))</span><br><span class="line">				<span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="built_in">len</span>(tokens)-<span class="number">1</span>)</span><br><span class="line">				<span class="keyword">if</span> tokens[i] <span class="keyword">in</span> <span class="string">&#x27;.?!&#x27;</span>]</span><br><span class="line"><span class="comment"># 进行训练，检验模型准确率</span></span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
+<h3 id="识别对话行为类型">识别对话行为类型</h3>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 提取即时消息会话语料库</span></span><br><span class="line">posts = nltk.corpus.nps_chat.xml_posts()[:<span class="number">10000</span>]</span><br><span class="line"><span class="comment"># 特征提取器，每个帖子包含什么词</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">dialogue_act_features</span>(<span class="params">post</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> word <span class="keyword">in</span> nltk.word_tokenize(post):</span><br><span class="line">    	features[<span class="string">&#x27;contains(%s)&#x27;</span> % word.lower()] = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"><span class="comment"># 构建模型，判断是否为对话行为</span></span><br><span class="line">featuresets = [(dialogue_act_features(post.text), post.get(<span class="string">&#x27;class&#x27;</span>)) <span class="keyword">for</span> post <span class="keyword">in</span> posts]</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
+<h3 id="识别文字蕴含">识别文字蕴含</h3>
+<p>识别文字蕴含（Recognizing textual entailment，RTE）是判断文本 T
+的一个给定片段是否蕴含着另一个叫做“假设”的文本。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">rte_features</span>(<span class="params">rtepair</span>):</span><br><span class="line">    <span class="comment">#  RTEFeatureExtractor 类建立了一个除去一些停用词后在文本和假设中都有的词汇包，然后计算重叠和差异</span></span><br><span class="line">    extractor = nltk.RTEFeatureExtractor(rtepair)</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    features[<span class="string">&#x27;word_overlap&#x27;</span>] = <span class="built_in">len</span>(extractor.overlap(<span class="string">&#x27;word&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;word_hyp_extra&#x27;</span>] = <span class="built_in">len</span>(extractor.hyp_extra(<span class="string">&#x27;word&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;ne_overlap&#x27;</span>] = <span class="built_in">len</span>(extractor.overlap(<span class="string">&#x27;ne&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;ne_hyp_extra&#x27;</span>] = <span class="built_in">len</span>(extractor.hyp_extra(<span class="string">&#x27;ne&#x27;</span>))</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line"><span class="comment"># 原文中rte识别率只是刚刚超过了58%，所以不多做介绍</span></span><br><span class="line">nltk.classify.rte_classify </span><br></pre></td></tr></table></figure>
+<h3 id="涉及到的定义">涉及到的定义</h3>
+<p>文中介绍一些关于评估模型、决策树、朴素贝叶斯分类器的知识。由于笔者已掌握这些基础模型知识所以在这里略去，以下介绍一些基本的定义。</p>
+<p>测试集：选择适当的比例，防止过拟合</p>
+<p>准确度：测量测试集上分类器正确标注的输入的比例。</p>
+<p>精确度（Precision），表示我们发现的项目中有多少是相关的，TP/(TP+
+FP)。</p>
+<p>召回率（Recall），表示相关的项目中我们发现了多少，TP/(TP+ FN)。</p>
+<p>F- 度量值（F-Measure）（或
+F-得分，F-Score），组合精确度和召回率为一个单独的得分，被定义为精确度和召回率的调和平均数(2
+× Precision × Recall)/(Precision+Recall)。</p>
+<p>混淆矩阵：其中每个 cells[i,j]表示正确的标签 i 被预测为标签 j 的次数
+。因此，对角线项目（即
+cells[i,i]）表示正确预测的标签，非对角线项目表示错误。nltk中可以使用
+<code>nltk.ConfusionMatrix</code>函数</p>
+<p>交叉验证：在不同的测试集上执行多个评估，然后组合这些评估的得分</p>
+<p>信息增益：用给定的特征分割输入值，衡量数据变得更有序的程度</p>
+<p>朴素贝叶斯假设（独立性假设）：每个输入值是通过首先为那个输入值选择一个类标签，然后产生每个特征的方式产生的
+，每个特征与其他特征完全独立。</p>
+<p>零计数：如果训练集中有特征从来没有和给定标签一起出现，导致给定标签的标签可能性为
+0。</p>
+<p>平滑：给定一个初值，解决零计数问题。<code>nltk.probability</code>提供了多种平滑技术</p>
+<p>Heldout估计使用一个heldout
+语料库计算特征频率与特征概率之间的关系。</p>
+<p>非二元特征：如果标签是1,2,4,5这种多元标签，可以转换0&lt;x&lt;3,
+3&lt;x&lt;6这种二元标签</p>
+<p>双重计数：在训练过程中特征的贡献被分开计算，但当使用分类器为新输入选择标签时，这些特征的贡献被组合。解决方案就是为每一个特征贡献设定一个权重。</p>
+<p>最大熵分类器：与朴素贝叶斯类似，使用搜索技术找出一组将最大限度地提高分类器性能的参数。避免使用广义
+迭代缩放（Generalized Iterative Scaling，GIS）或改进的迭代缩放（Improved
+Iterative Scaling，IIS），这两者都比共轭梯度（Conjugate Gradient，CG
+）和 BFGS 优化方法慢很多</p>
+<p>联合特征：每个接收它自己的参数的标签和特征的组合。联合特征是有标签的的值的属性，而（简单）特征是未加标签的值的属性。</p>
+<p>一般情况下，
+最大熵原理是说在与我们所知道的一致的的分布中，我们会选择熵最高（分布最均匀）的。</p>
+<p>对于每个联合特征，最大熵模型计算该特征的“经验频率”——即它出现在训练集中的频率。然后，它搜索熵最大的分布，同时也预测每个联合特征正确的频率。</p>
+<p>一般情况下，生成式模型确实比条件式模型强大。但是前者所需的参数会大于后者。</p>
 ]]></content>
       <categories>
-        <category>Note</category>
+        <category>NLP</category>
       </categories>
       <tags>
-        <tag>数学</tag>
+        <tag>Python</tag>
+        <tag>笔记</tag>
+        <tag>NLTK</tag>
       </tags>
   </entry>
   <entry>
-    <title>NLTK阅读笔记Ⅰ</title>
-    <url>/hexo_blog/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/</url>
-    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
+    <title>最优化算法学习记录</title>
+    <url>/hexo_blog/2021/02/03/%E6%95%B0%E5%AD%A6/%E6%9C%80%E4%BC%98%E5%8C%96%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/</url>
+    <content><![CDATA[<p>本文提供了优化问题的概述，包括背景，分类，以及线性，非线性和二次规划问题的求解方法。本文还讨论了全局优化，并提供了非凸函数和凸函数的例子，以及线性规划和二次规划的例子。</p>
 <span id="more"></span>
-<h2 id="写在开头">写在开头</h2>
-<p>在这里主要是对NLP的相关知识做一个整理和对NLTK模块的介绍，书中Python基础内容在这里不做介绍，如书中有我认为值得介绍的Python写法，我会进行说明。</p>
-<p>由于书中nltk是老版本的问题，语法上存在一些变动，在这里也会进行修正。可能由于书是译本的原因（当然也可能原作者自己的失误），部分代码有些错误，在这里我也进行了校正。如果您发现了我的文章中有需要改正或改进的地方，欢迎在评论区提出。</p>
-<p>最后十分感谢原作者的贡献和译者的翻译，强烈推荐读者亲自阅读此书。</p>
-<p>nltk版本是3.2.3，<a href="http://www.nltk.org/">官网</a></p>
-<p>python版本3.6.1</p>
-<p>数据、PDF和一些重要的代码已经放在了Github上了。<a
-href="https://github.com/wnma3mz/Nltk_Study">github地址</a></p>
-<h2 id="第一章语言处理与python">第一章：语言处理与Python</h2>
-<p>NLTK库的安装，在这里不做介绍，强烈建议直接使用Anaconda环境。<a
-href="https://www.anaconda.com/download/">官网下载链接</a>。本文是直接使用了Anaconda环境。</p>
-<p>数据的获取，这里nltk有官方提供的文本数据，可以直接使用<code>nltk.download()</code>打开图形界面，下载语料集<code>book</code>。由于使用这个方法下载速度比较慢的原因，我在这里Github上提供了数据集<code>nltk_data</code>，下载之后，移动到<code>nltk.download()</code>原本的下载目录下，之后再运行<code>nltk.download()</code>就不需要下载数据集而是直接解压数据集，速度会快上很多。<a
-href="https://github.com/wnma3mz/Nltk_Study">下载链接</a></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导入nltk模块</span></span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="comment"># 导入基本语料集(不需要额外下载)，包含text1到text9变量，可以直接输出这些变量</span></span><br><span class="line"><span class="keyword">from</span> nltk.book <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索文本。这里表示找到&quot;monstrous&quot;所包含的句子，并且输出上下文</span></span><br><span class="line">text1.concordance(<span class="string">&quot;monstrous&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索文本出现在相似的上下文中</span></span><br><span class="line">text1.similar(<span class="string">&quot;monstrous&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 搜索两个及两个以上共同词的上下文</span></span><br><span class="line">text2.common_contexts([<span class="string">&quot;monstrous&quot;</span>, <span class="string">&quot;very&quot;</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 画一张离散图表示这些词出现在文本中的位置，输出见下图</span></span><br><span class="line">text4.dispersion_plot([<span class="string">&quot;citizens&quot;</span>, <span class="string">&quot;democracy&quot;</span>, <span class="string">&quot;freedom&quot;</span>, <span class="string">&quot;duties&quot;</span>, <span class="string">&quot;America&quot;</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 基于文本，随机生成一些文本</span></span><br><span class="line">text3.generate()</span><br></pre></td></tr></table></figure>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-1.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 有序字典，按词频从高到低排序</span></span><br><span class="line">fdist1 = FreqDist(text1)</span><br><span class="line"><span class="comment"># 选出词频最高的50个词</span></span><br><span class="line">fdist1.keys()[:<span class="number">50</span>]</span><br><span class="line"><span class="comment"># 某个词出现的频数</span></span><br><span class="line">fdist1[<span class="string">&#x27;whale&#x27;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># text1中词频最高的50个单词，进行绘图，输出见下图</span></span><br><span class="line">fdist1.plot(<span class="number">50</span>, cumulative=<span class="literal">True</span>)</span><br><span class="line"><span class="comment"># text1中只出现过一次的单词</span></span><br><span class="line">fdist1.hapaxes()</span><br></pre></td></tr></table></figure>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-3.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 词语搭配，双连词(bigrams)</span></span><br><span class="line">nltk.bigrams([<span class="string">&#x27;more&#x27;</span>,<span class="string">&#x27;is&#x27;</span>, <span class="string">&#x27;said&#x27;</span>, <span class="string">&#x27;than&#x27;</span>, <span class="string">&#x27;done&#x27;</span>])</span><br><span class="line"><span class="comment"># 输出</span></span><br><span class="line">[(<span class="string">&#x27;more&#x27;</span>,<span class="string">&#x27;is&#x27;</span>),(<span class="string">&#x27;is&#x27;</span>,<span class="string">&#x27;said&#x27;</span>),(<span class="string">&#x27;said&#x27;</span>, <span class="string">&#x27;than&#x27;</span>), (<span class="string">&#x27;than&#x27;</span>,<span class="string">&#x27;done&#x27;</span>)]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 文本中单个词的频率预期得到的更频繁出现的双连词</span></span><br><span class="line">text4.collocations()</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 与nltk自带的聊天机器人系统对话</span></span><br><span class="line"><span class="comment"># 导入模块</span></span><br><span class="line"><span class="keyword">from</span> nltk.chat <span class="keyword">import</span> chatbots</span><br><span class="line"><span class="comment"># 选择一个聊天机器人并开始对话</span></span><br><span class="line">chatbots()</span><br></pre></td></tr></table></figure>
-<p>补充部分：</p>
-<table>
-<colgroup>
-<col style="width: 7%" />
-<col style="width: 92%" />
-</colgroup>
-<thead>
-<tr>
-<th style="text-align: center;">text</th>
-<th style="text-align: center;">文本名</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style="text-align: center;">text1</td>
-<td style="text-align: center;">《白鲸记》（Moby Dick by Herman Melville
-1851）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text2</td>
-<td style="text-align: center;">《理智与情感》（Sense and Sensibility by
-Jane Austen 1811）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text3</td>
-<td style="text-align: center;">《创世纪》（The Book of Genesis）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text4</td>
-<td style="text-align: center;">《就职演说语料库》（Inaugural Address
-Corpus）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text5</td>
-<td style="text-align: center;">《NPS聊天语料库》（Chat Corpus）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text6</td>
-<td style="text-align: center;">《巨蟒与圣杯》（Monty Python and the
-Holy Grail）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text7</td>
-<td style="text-align: center;">《华尔街日报》（Wall Street
-Journal）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text8</td>
-<td style="text-align: center;">《个人文集》（Personals Corpus）</td>
-</tr>
-<tr>
-<td style="text-align: center;">text9</td>
-<td style="text-align: center;">《周四的男人》（The Man Who Was Thursday
-by G . K . Chestert）</td>
-</tr>
-</tbody>
-</table>
-<p>概念补充：</p>
+<p>优化问题的最终目的就是求最小值（最大值转为求最小值）</p>
+<h3 id="简单划分">简单划分</h3>
+<ol type="1">
+<li>解析法</li>
+<li><strong>迭代法</strong>
+<ol type="1">
+<li>常规方法</li>
+<li>技巧
+<ol type="1">
+<li>切平面</li>
+<li>椭圆法</li>
+<li>内点法</li>
+</ol></li>
+</ol></li>
+</ol>
+<h3 id="background">Background</h3>
+<ol type="1">
+<li>目标函数、约束、线性（非线性）</li>
+<li>凸优化问题（是否为凸问题）</li>
+<li>局部最优、全局最优</li>
+<li>矩阵求导（梯度），矩阵的一阶导数、二阶导数（Hessian）</li>
+<li>解析与迭代</li>
+</ol>
+<h3 id="how">How</h3>
+<ol type="1">
+<li>目标函数、约束均为线性：单纯形法。大问题：内点法</li>
+<li>目标函数二次，约束线性：改进单纯形法。大问题：内点法</li>
+<li>凸问题：切割平面、椭球算法、<strong>内点法</strong></li>
+<li>目标函数存在多个局部最小值，全局优化算法（启发式算法/智能优化算法）</li>
+<li>非线性非凸问题（最难），梯度和海森信息（二阶导）-&gt;构造解析表达式</li>
+<li>无约束问题
+<ol type="1">
+<li>Levenberg-Marquardt</li>
+<li>牛顿算法</li>
+<li>拟牛顿</li>
+<li>最速下降</li>
+<li>Powell</li>
+</ol></li>
+<li>线性约束：SQP、梯度投影法</li>
+<li>非线性约束：SQP、转为为无约束</li>
+</ol>
+<h3 id="优化问题分类">优化问题分类</h3>
+<ol type="1">
+<li><p>无约束问题（非线性最小二乘问题）</p>
+<ol type="1">
+<li>牛顿法</li>
+<li>逆牛顿法</li>
+<li>Levenberg-Marquardt算法</li>
+<li>Nelder-Mead 方法</li>
+<li>其他
+<ol type="1">
+<li>搜索方向和一维搜索法</li>
+</ol></li>
+</ol></li>
+<li><p>约束问题</p>
+<ol type="1">
+<li>规划问题
+<ol type="1">
+<li>线性规划（整数规划）
+<ol type="1">
+<li>单纯形解法</li>
+</ol></li>
+<li>二次规划
+<ol type="1">
+<li>改进单纯形法</li>
+</ol></li>
+</ol></li>
+<li>非线性优化的约束
+<ol type="1">
+<li>等式约束
+<ol type="1">
+<li>线性等式约束</li>
+<li>非线性等式约束</li>
+</ol></li>
+<li>不等式约束
+<ol type="1">
+<li>线性等式约束</li>
+<li>非线性等式约束</li>
+</ol></li>
+</ol></li>
+</ol></li>
+</ol>
+<h3 id="全局优化">全局优化</h3>
+<ol type="1">
+<li>启发式算法
+<ol type="1">
+<li>模拟退火</li>
+<li>遗传算法</li>
+</ol></li>
+<li>基本算法
+<ol type="1">
+<li>随机搜索</li>
+<li>多起点优化</li>
+</ol></li>
+</ol>
+<h3 id="优化问题fx">优化问题<span
+class="math inline">\(F(x)\)</span></h3>
 <ul>
-<li>词类型：一个词在一个文本中独一无二的出现形式或拼写。也就是说，这个词在词汇表中是唯一的。</li>
-<li>频率分布：</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-2.png" /></p>
+<li>线性问题
 <ul>
-<li>FreqDist的API</li>
-</ul>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-4.png" /></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/1-5.png" /></p>
+<li>不论有无约束，都可以使用<strong>单纯形法</strong>和<strong>内点法</strong>求解</li>
+</ul></li>
+<li>非线性问题
 <ul>
-<li>词意消歧：需要算出特定上下文中的词被赋予的是哪个意思。单词可能存在相同/相近的含义，此时需要根据上下文来推断单词在此情景下的含义。</li>
-<li>指代消解（anaphora resolution）：确定代词或名词短语指的是什么</li>
-<li>先行词：代词可能代表的对象</li>
-<li>语义角色标注（semantic role
-labeling）：确定名词短语如何与动词相关联（如施事，受事，工具等）</li>
-<li>自动生成语言：需要解决自动语言理解。弄清楚词的含义、动作的主语以及代词的先行词是理解句子含义的步骤</li>
-<li>机器翻译（MT）：难点，一个给定的词可能存在几种不同的解释。</li>
-<li>文本对齐：根据一个网站发布的多种语言版本，来自动配对组成句子</li>
-<li>人机对话系统：图灵测试。</li>
-<li>文本含义识别（Recognizing Textual Entailment 简称
-RTE）：根据假定的一些条件，来推断给出的一句话是否正确。</li>
-</ul>
-<h2
-id="第二章获得文本语料和词汇资源">第二章：获得文本语料和词汇资源</h2>
-<p><a href="http://www.gutenberg.org/">古腾堡语料库</a></p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-1.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 导入nltk的古腾堡语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> gutenberg</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看这个语料库中的所有txt文件名</span></span><br><span class="line">gutenberg.fileids()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选中其中的一个</span></span><br><span class="line">emma = gutenberg.words(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br><span class="line"><span class="comment"># 查看它包含的词汇个数</span></span><br><span class="line"><span class="built_in">len</span>(emma)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 如果要使用第一章中concordance()这样的命令，就必须要将数据放到nltk.Text对象中</span></span><br><span class="line">emma = nltk.Text(emma)</span><br><span class="line">emma.concordance(<span class="string">&quot;suprprize&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用words获取的是经过分隔成标识符的文本。使用sents函数则可以获取以句子进行划分后的数据。具体区别可以运行看看，请读者自行对比。</span></span><br><span class="line">emma_sents = gutenberg.sents(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取原始文本。一个字符串</span></span><br><span class="line">emma_raw = gutenberg.raw(<span class="string">&#x27;austen-emma.txt&#x27;</span>)</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 网络和聊天文本</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> webtext</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> nps_chat</span><br><span class="line"></span><br><span class="line"><span class="comment"># 布朗语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"></span><br><span class="line"><span class="comment"># 布朗语料库中所有的类别</span></span><br><span class="line">brown.categories()</span><br><span class="line"><span class="comment"># words函数根据类别选择数据</span></span><br><span class="line">brown.words(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">[<span class="string">&#x27;The&#x27;</span>, <span class="string">&#x27;Fulton&#x27;</span>, <span class="string">&#x27;County&#x27;</span>, <span class="string">&#x27;Grand&#x27;</span>, <span class="string">&#x27;Jury&#x27;</span>, <span class="string">&#x27;said&#x27;</span>, ...]</span><br><span class="line"><span class="comment"># words函数根据文件名选择数据</span></span><br><span class="line">brown.words(fileids=[<span class="string">&#x27;cg22&#x27;</span>])</span><br><span class="line">[<span class="string">&#x27;Does&#x27;</span>, <span class="string">&#x27;our&#x27;</span>, <span class="string">&#x27;society&#x27;</span>, <span class="string">&#x27;have&#x27;</span>, <span class="string">&#x27;a&#x27;</span>, <span class="string">&#x27;runaway&#x27;</span>, <span class="string">&#x27;,&#x27;</span>, ...]</span><br><span class="line"><span class="comment"># sents函数根据类别选择数据</span></span><br><span class="line">brown.sents(categories=[<span class="string">&#x27;news&#x27;</span>, <span class="string">&#x27;editorial&#x27;</span>, <span class="string">&#x27;reviews&#x27;</span>])</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="comment"># 带条件的频率分布函数</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 取出brown语料库中所有类别中的所有词。每个类别对应该类别下的所有词。(类别，所有词)</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (genre, word)</span><br><span class="line">        <span class="keyword">for</span> genre <span class="keyword">in</span> brown.categories()</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> brown.words(categories=genre))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 指定一部分文本类别(genres)</span></span><br><span class="line">genres = [<span class="string">&#x27;news&#x27;</span>, <span class="string">&#x27;religion&#x27;</span>, <span class="string">&#x27;hobbies&#x27;</span>, <span class="string">&#x27;science_fiction&#x27;</span>, <span class="string">&#x27;romance&#x27;</span>, <span class="string">&#x27;humor&#x27;</span>]</span><br><span class="line"><span class="comment"># 指定一些情态动词(modals)</span></span><br><span class="line">modals = [<span class="string">&#x27;can&#x27;</span>, <span class="string">&#x27;could&#x27;</span>, <span class="string">&#x27;may&#x27;</span>, <span class="string">&#x27;might&#x27;</span>, <span class="string">&#x27;must&#x27;</span>, <span class="string">&#x27;will&#x27;</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出一张表格。genres与modals的对应数量关系</span></span><br><span class="line">cfd.tabulate(conditions=genres, samples=modals)</span><br><span class="line"><span class="comment"># 输出如下</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">                  can could   may might  must  will</span></span><br><span class="line"><span class="string">           news    93    86    66    38    50   389</span></span><br><span class="line"><span class="string">       religion    82    59    78    12    54    71</span></span><br><span class="line"><span class="string">        hobbies   268    58   131    22    83   264</span></span><br><span class="line"><span class="string">science_fiction    16    49     4    12     8    16</span></span><br><span class="line"><span class="string">        romance    74   193    11    51    45    43</span></span><br><span class="line"><span class="string">          humor    16    30     8     8     9    13</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span></span><br></pre></td></tr></table></figure>
-<p>通过输出，我们很容易发现，在<code>news</code>类别里面用的最多的情态动词是<code>will</code>。当然，还有一些结论就由读者自行发现了。</p>
-<p>关于其他语料库的信息挖掘，在这里就不赘述了。书中也只是浅显的用了之前用过的一些方法。下面展示一些我认为比较有意思的代码及输出。</p>
-<p>官网提供了如何访问NLTK语料库的其他例子，<a
-href="http://www.nltk.org/howto/corpus.html">链接</a></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> inaugural</span><br><span class="line"></span><br><span class="line"><span class="comment"># 先选出inaugural语料库中的所有字段的对应所有词汇，再统计出文本中以`america`和`citizen`开头的词在每个年份（filed）出现次数。（`america`或`citizen`, 年份）</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (target, fileid[:<span class="number">4</span>])</span><br><span class="line">        <span class="keyword">for</span> fileid <span class="keyword">in</span> inaugural.fileids()</span><br><span class="line">        <span class="keyword">for</span> w <span class="keyword">in</span> inaugural.words(fileid)</span><br><span class="line">        <span class="keyword">for</span> target <span class="keyword">in</span> [<span class="string">&#x27;america&#x27;</span>, <span class="string">&#x27;citizen&#x27;</span>]</span><br><span class="line">        <span class="keyword">if</span> w.lower().startswith(target))</span><br><span class="line"><span class="comment"># 绘图</span></span><br><span class="line">cfd.plot()</span><br></pre></td></tr></table></figure>
-<p>以<code>america</code>或<code>citizen</code>开始的词随时间（年份）的演变趋势</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-2.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># udhr语料库中不同语言版本的字长差异（彩色图）</span></span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> udhr</span><br><span class="line"></span><br><span class="line"><span class="comment"># 选取比较的语言</span></span><br><span class="line">languages = [<span class="string">&#x27;Chickasaw&#x27;</span>, <span class="string">&#x27;English&#x27;</span>, <span class="string">&#x27;German_Deutsch&#x27;</span>, <span class="string">&#x27;Greenlandic_Inuktikut&#x27;</span>, <span class="string">&#x27;Hungarian_Magyar&#x27;</span>, <span class="string">&#x27;Ibibio_Efik&#x27;</span>]</span><br><span class="line"><span class="comment"># 提取不同语言的数据</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(</span><br><span class="line">        (lang, <span class="built_in">len</span>(word))</span><br><span class="line">        <span class="keyword">for</span> lang <span class="keyword">in</span> languages</span><br><span class="line">        <span class="keyword">for</span> word <span class="keyword">in</span> udhr.words(lang + <span class="string">&#x27;-Latin1&#x27;</span>))</span><br><span class="line"></span><br><span class="line">cfd.plot(cumulative=<span class="literal">True</span>)</span><br></pre></td></tr></table></figure>
-<p>累积字长分布： “世界人权宣言”的6个翻译版本 <img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-3.png" /></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 载入自己的语料库，文本文件</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> PlaintextCorpusReader</span><br><span class="line"><span class="comment"># 设置语料库的路径</span></span><br><span class="line">corpus_root = <span class="string">&#x27;/usr/share/dict&#x27;</span></span><br><span class="line"><span class="comment"># 解析路径下的所有文件</span></span><br><span class="line">wordlists = PlaintextCorpusReader(corpus_root, <span class="string">&#x27;.*&#x27;</span>)</span><br><span class="line"><span class="comment"># 语料库中所有的文件名</span></span><br><span class="line">wordlists.fileids()</span><br><span class="line"><span class="comment"># 语料库中`connectives`中的单词</span></span><br><span class="line">wordlists.words(<span class="string">&#x27;connectives&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 本地个人的语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> BracketParseCorpusReader</span><br><span class="line">corpus_root = <span class="string">r&quot;C:\corpora\penntreebank\parsed\mrg\wsj&quot;</span></span><br><span class="line"><span class="comment"># 使用正则匹配</span></span><br><span class="line">file_pattern = <span class="string">r&quot;.*/wsj_.*\.mrg&quot;</span></span><br><span class="line">ptb = BracketParseCorpusReader(corpus_root, file_pattern)</span><br><span class="line">ptb.fileids()</span><br></pre></td></tr></table></figure>
-<p>双连词运用 <figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 依次连续输出最有可能跟在上一个词后面的词</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">generate_model</span>(<span class="params">cfdist, word, num=<span class="number">15</span></span>):</span><br><span class="line">    <span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(num):</span><br><span class="line">        <span class="built_in">print</span>(word)</span><br><span class="line">        word = cfdist[word].<span class="built_in">max</span>()</span><br><span class="line"><span class="comment"># 选中语料库，提取文本</span></span><br><span class="line">text = nltk.corpus.genesis.words(<span class="string">&#x27;english-kjv.txt&#x27;</span>)</span><br><span class="line"><span class="comment"># 生成双连词</span></span><br><span class="line">bigrams = nltk.bigrams(text)</span><br><span class="line"><span class="comment"># 创建双连词频数对象</span></span><br><span class="line">cfd = nltk.ConditionalFreqDist(bigrams)</span><br><span class="line"><span class="comment"># 找出最有可能在`living`后面的单词。</span></span><br><span class="line"><span class="built_in">print</span>(cfd[<span class="string">&#x27;living&#x27;</span>])</span><br><span class="line">generate_model(cfd, <span class="string">&#x27;living&#x27;</span>)</span><br></pre></td></tr></table></figure></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 加载停用词语料库</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> stopwords</span><br><span class="line"><span class="comment"># 加载英语中的停用词</span></span><br><span class="line">stopwords.words(<span class="string">&#x27;English&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载单词列表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> words</span><br><span class="line">words.words()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载人名表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> names</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载发音词典</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> cmudict</span><br><span class="line">cmudict.entries()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载比较词表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> swadesh</span><br><span class="line"><span class="comment"># 比较词表的标识码</span></span><br><span class="line">swadesh.fileids()</span><br><span class="line"><span class="comment"># 输出比较词汇中的`en`文本资料</span></span><br><span class="line">swadesh.words(<span class="string">&#x27;en&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 加载词汇工具</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> toolbox</span><br><span class="line"><span class="comment"># 输出罗托卡特语（Rotokas）的词典文本资料</span></span><br><span class="line">toolbox.entries(<span class="string">&#x27;rotokas.dic&#x27;</span>)</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 加载WordNet</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> wordnet <span class="keyword">as</span> wn</span><br><span class="line"><span class="comment"># 输出`motocar`的同义词</span></span><br><span class="line">wn.synsets(<span class="string">&#x27;motorcar&#x27;</span>)</span><br><span class="line"><span class="comment"># 输出`car`名词的第一个意义中的同义词集</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).lemma_names</span><br><span class="line"><span class="comment"># 输出`car`名词的第一个意义的定义</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).definition</span><br><span class="line"><span class="comment"># 输出同义词集的所有词条</span></span><br><span class="line">wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>).lemmas</span><br><span class="line"><span class="comment"># 输出特定的词条</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>)</span><br><span class="line"><span class="comment"># 输出一个词条对应的同义词集</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>).synset</span><br><span class="line"><span class="comment"># 输出一个词条的&quot;名字&quot;</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;car.n.01.automobile&#x27;</span>).name</span><br><span class="line"><span class="comment"># 输出所有包含词`car`的词条</span></span><br><span class="line">wn.lemmas(<span class="string">&#x27;car&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取cat词性为名词的意义</span></span><br><span class="line">motorcar = wn.synset(<span class="string">&#x27;car.n.01&#x27;</span>)</span><br><span class="line"><span class="comment"># 获取motorcar的下位词列表</span></span><br><span class="line">types_of_motorcar = motorcar.hyponyms()</span><br><span class="line"><span class="comment"># 获取motorcar的上位词列表</span></span><br><span class="line">motorcar.hypernyms()</span><br><span class="line"><span class="comment"># 获取motorcar上位词的路径列表</span></span><br><span class="line">paths = motorcar.hypernym_paths()</span><br><span class="line"><span class="comment"># 得到motorcar的根上位（最一般的上位）同义词集</span></span><br><span class="line">motorcar.root_hypernyms()</span><br><span class="line"><span class="comment"># 打开图形化WordNet的浏览器，更加便捷探索关系</span></span><br><span class="line">nltk.app.wordnet()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 更多词汇关系</span></span><br><span class="line"><span class="comment"># 整体到部分：想象WordNet网络关系是一棵树</span></span><br><span class="line"><span class="comment"># 得到树的一部分，如树干、树冠等</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).part_meronyms()</span><br><span class="line"><span class="comment"># 得到树的实质，即心材和边材</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).substance_meronyms()</span><br><span class="line"><span class="comment"># 得到许多树构成的集合，即森林</span></span><br><span class="line">wn.synset(<span class="string">&#x27;tree.n.01&#x27;</span>).member_holonyms()</span><br><span class="line"><span class="comment"># 得到mint所有名词意义的列表</span></span><br><span class="line">wn.synsets(<span class="string">&#x27;mint&#x27;</span>, wn.NOUN)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 想象走路的动作</span></span><br><span class="line"><span class="comment"># 找出走路的一部分动作：抬脚</span></span><br><span class="line">wn.synset(<span class="string">&#x27;walk.v.01&#x27;</span>).entailments()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 反义词</span></span><br><span class="line">wn.lemma(<span class="string">&#x27;supply.n.02.supply&#x27;</span>).antonyms()</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用dir()查看词汇关系和同义词集上的定义</span></span><br><span class="line"><span class="built_in">dir</span>(wn.synset(<span class="string">&#x27;harmony.n.02&#x27;</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 语义相似度</span></span><br><span class="line"><span class="comment"># 初始化一个同义词集中的两个单词</span></span><br><span class="line">right = wn.synset(<span class="string">&#x27;right_whale.n.01&#x27;</span>)</span><br><span class="line">minke = wn.synset(<span class="string">&#x27;minke_whale.n.01&#x27;</span>)</span><br><span class="line"><span class="comment"># 如果它们有共同的上位词（且层次结构较低），则它们的关系一定十分密切</span></span><br><span class="line">right.lowest_common_hypernyms(minke)</span><br><span class="line"><span class="comment"># 查找同义词集深度量化关系，输出为数值。越小表示离根节点越近</span></span><br><span class="line">wn.synset(<span class="string">&#x27;baleen_whale.n.01&#x27;</span>).min_depth()</span><br><span class="line"><span class="comment"># 两个词 基于上位词层次结构中相互连接的概念之间的最短路径在0-1范围的打分（二者没有关系就返回1）</span></span><br><span class="line">right.path_similarity(minke)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 其他相似性度量方法，可以通过help(wn)获取。比如:VerbNet</span></span><br></pre></td></tr></table></figure>
-<p>概念补充：</p>
+<li>凸优化问题
+<ul>
+<li>有约束：切平面法、椭球法、内点法</li>
+<li>无约束情况下等同于非凸问题</li>
+</ul></li>
+<li>非凸问题
+<ul>
+<li>约束<span
+class="math inline">\(h(x)\)</span>不是线性，拉格朗日乘子法</li>
+<li>约束<span class="math inline">\(g(x)\)</span>为线性，gradient
+projection、SQP</li>
+<li>约束<span
+class="math inline">\(g(x)\)</span>不是线性，惩罚函数、Barrier函数、SQP问题</li>
+<li>无约束（可消约束）
 <ul>
-<li><p>文本语料库的结构：</p>
+<li>多起点优化
 <ul>
-<li><p>孤立的无特别组织的文本集合</p></li>
-<li><p>按文体等分类组织结构</p></li>
-<li><p>分类重叠，主题类别（路透社语料库）</p></li>
-<li><p>随时间变化语言语言用法的改变（就职演说语料库）</p></li>
+<li>Hession矩阵，一阶矩阵：Levenberg-Marquardt、牛顿法、牛顿线性搜索</li>
+<li>一阶矩阵：Quasi-Newton、Quasi-Newton线性搜索、最速下降法</li>
+<li>无：Powell Perpendicular线性搜索、Nelder-Mead</li>
+</ul></li>
+<li>不是多起点优化：随机搜索、模拟退火、遗传算法</li>
+</ul></li>
+</ul></li>
+</ul></li>
+<li>二次问题
+<ul>
+<li>约束，线性，修正的单纯形法、内点法</li>
 </ul></li>
-<li><p>NLTK中定义的基本语料库函数</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-4.png" /></p></li>
-<li><p>条件频率分布</p>
-<p><strong>条件频率分布是频率分布的集合，每个频率分布有一个不同的“条件”。这个条件通常是文本的类别。</strong></p>
-<p>当语料文本被分为几类（文体、主题、作者等）时，我们可以计算每个类别独立的频率分布，研究类别之间的系统性差异。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-5.png" /></p></li>
-<li><p>条件和事件</p>
-<p>对于NLP来说，文本出现的词汇就是事件。条件频率分布需要给每个事件关联一个条件，所以处理的是一个配对序列。每对形式是：（条件，事件）。举例就是，（文本类别，文本中的一个词汇）</p></li>
-<li><p>FreqDist()以一个简单的链表作为输入，ConditionalFreqDist()以一个配对链表作为输入。</p></li>
-<li><p>停用词：
-如<code>the</code>、<code>to</code>这种高频词汇。这种词通常没有什么词汇内容，但是它们又会让文本分析变得困难。所以在需要的情况下，我们就需要从文档中过滤掉他们。</p></li>
-<li><p>WordNet：面向语义的英语词典，类似与传统辞典，但具有更丰富的结构。每个节点对应一个<strong>同义词集</strong>，边表示上/下位词关系（上下级概念与从属概念的关系）</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/2-6.png" /></p></li>
-<li><p>synset：同义词集，意义想相同的词（或“词条”）的集合。如car.n.01是car的第一个名词的意义，就是同义词集。</p></li>
-<li><p>词条：同义词集和词的配对。</p></li>
 </ul>
-<p>下一篇：<a href="/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/" title="NLTK阅读笔记Ⅱ">NLTK阅读笔记Ⅱ</a></p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/optimization/%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95%E7%9A%84%E9%80%89%E6%8B%A9.png" />
+<img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/optimization/%E6%9C%80%E4%BC%98%E5%8C%96%E7%AE%97%E6%B3%95%E5%81%9C%E6%AD%A2%E5%87%86%E5%88%99.png" /></p>
+<h3 id="例子">例子</h3>
+<p>非凸函数</p>
+<p><span class="math inline">\(\int _ { 0 } ^ { x 2 } ( \frac { e ^ { x
+_ { 3 } ^ { 2 } z } \cos ( z ) } { x _ { 3 } ^ { 4 } z + 1 } + z ^ { x 1
+} ) d z\)</span></p>
+<p><span class="math inline">\(e ^ { x _ { 1 } ^ { 2 } + x _ { 2 } ^ { 2
+} } + x _ { 1 } ^ { 2 } x _ { 2 } ^ { 2 }\)</span></p>
+<p><span class="math inline">\(( x _ { 1 } + x _ { 2 } ) ^ { 3 } - \log
+( 2 - x _ { 1 } ^ { 2 } - x _ { 2 } ^ { 2 } )\)</span></p>
+<p>$x _ { 1 } ^ { 4 } + x _ { 2 } ^ { 2 } + 3 x _ { 1 } x _ { 3 } + 2 x
+_ { 4 } x _ { 5 } $</p>
+<p>$9 x _ { 1 } ^ { 2 } + 4 x _ { 2 } ^ { 2 } + 2 x _ { 1 } x _ { 2 } +
+8 x _ { 3 } ^ { 2 } - 2 x _ { 1 } x _ { 3 } $</p>
+<p>$ { ( x _ { 1 } + 3 x _ { 2 } + 4 x _ { 3 } ) ^ { 2 } } + ( ( x _ { 1
+} + 4 x _ { 2 } + 5 x _ { 3 } - 100 ) ^ { 4 } )$</p>
+<p>$ { 1 + } $</p>
+<p>凸函数</p>
+<p>$( x _ { 1 } ^ { 2 } + x _ { 2 } ^ { 2 } ) + 14 x _ { 2 } ^ { 4 }
+$</p>
+<p>$ ( ( x _ { 1 } - 1 ) ^ { 2 } + ( x _ { 2 } - 2 ) ^ { 2 } ) ^ { 2 } -
+( 2 - x _ { 1 } ^ { 2 } - x _ { 2 } ^ { 2 } )$</p>
+<p>$| 2 x _ { 1 } + 4 x _ { 2 } + 3 x _ { 3 } + 2 x _ { 4 } + 7 | + ( (
+x _ { 1 } + x _ { 2 } - 4 x _ { 3 } - x _ { 4 } ) ^ { 4 } ) $</p>
+<p>$( F _ { 1 } ( x ) , 2 F _ { 2 } ( x ) , F _ { 3 } ( x ) + 8 F _ { 4
+} ( x ) ) $</p>
+<p>$ x _ { 1 } + | x _ { 2 } | + 3 x _ { 3 } + 4 | x _ { 4 } |$</p>
+<p><span class="math inline">\(( 4 x _ { 1 } + 7 x _ { 2 } - 4 x _ { 3 }
++ 12 x _ { 4 } ) ^ { 3 }\)</span></p>
+<p><span class="math inline">\(( x _ { 1 } - 5 ) ^ { 2 } + ( x _ { 2 } -
+8 ) ^ { 4 }\)</span></p>
+<p>线性规划问题</p>
+<p><span class="math inline">\(| x _ { 1 } - 4 x _ { 2 } + x _ { 3 } - 4
+|\)</span></p>
+<p>$( 4 x _ { 1 } - 3 x _ { 2 } + 8 x _ { 3 } - 5 , - 2 x _ { 1 } + 7 x
+_ { 2 } - x _ { 3 } + 1 ) $</p>
+<p>二次规划（凸二次函数）</p>
+<p><span class="math inline">\(2 ^ { 5 x _ { 1 } ^ { 2 } + 2 x _ { 2 } ^
+{ 2 } + 6 x _ { 1 } x _ { 2 } + 3 x _ { 1 } - x _ { 2 } }\)</span></p>
+<p><span class="math inline">\(1 - 4 x _ { 1 } ^ { 2 } - 9 x _ { 2 } ^ {
+2 } - 8 x _ { 3 } ^ { 2 } + 4 x _ { 1 } x _ { 2 } - 2 x _ { 1 } x _ { 3
+} + 4 x _ { 2 } x _ { 3 }\)</span></p>
+<h3 id="相关书籍">相关书籍</h3>
+<p>[1] 程理民, 吴江, 张玉林. 运筹学模型与方法教程[M]. 清华大学出版社,
+2000.</p>
+<p>[2] 陈宝林. 最优化理论与算法[M]. 清华大学出版社有限公司, 2005.</p>
+<p>[3] 袁亚湘, 孙文瑜. 最优化理论与方法[M]. 科学出版社, 1997.</p>
+<p>个人感觉，难度[3]&gt;[2]&gt;[1]，袁院士那本理论证明偏多，可搭配[2]与课后练习一起阅读。</p>
 ]]></content>
       <categories>
-        <category>NLP</category>
+        <category>Algorithm</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>笔记</tag>
-        <tag>NLTK</tag>
+        <tag>Optimization</tag>
       </tags>
   </entry>
   <entry>
-    <title>NLTK阅读笔记Ⅱ</title>
-    <url>/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/</url>
-    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
-<p>接 <a href="/hexo_blog/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/" title="NLTK阅读笔记Ⅰ">NLTK阅读笔记Ⅰ</a></p>
+    <title>神经网络笔记Ⅱ</title>
+    <url>/hexo_blog/2018/08/20/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A1/</url>
+    <content><![CDATA[<p>接 <a href="/hexo_blog/2018/04/14/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A0/" title="神经网络笔记Ⅰ">神经网络笔记Ⅰ</a></p>
+<p>从CNN到RNN再到LSTM，深入了解三种神经网络。其中，CNN主要用于图像识别与处理，RNN处理具有动态时序行为的任务，而LSTM则在此基础上添加了记忆单元，更适合处理时间序列中间隔和延迟非常长的事件。仅靠前向传播已不满足要求，反向传播也变得重要。</p>
 <span id="more"></span>
-<h2 id="第三章加工原料文本">第三章：加工原料文本</h2>
-<p><a href="http://www.gutenberg.org/catalog/">古腾堡项目</a></p>
-<p>可以从这个项目中获取感兴趣的文本，之后进行分析。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="comment"># 将原始文本字符串转换为list，以\r\n标点符号等为分隔符进行切割</span></span><br><span class="line">tokens = nltk.word_tokenize(raw)</span><br><span class="line"><span class="comment"># 创建NLTK文本</span></span><br><span class="line">text = nltk.Text(tokens)</span><br></pre></td></tr></table></figure>
-<p>处理文本部分：</p>
-<p><code>requests</code>： 数据获取</p>
-<p><code>BeautifulSoup</code>、<code>re</code>、<code>xpath</code>：
-文本处理</p>
-<p><code>feedparser</code>：处理RSS订阅</p>
-<p><code>os</code>：处理本地文件</p>
-<p>字符串基本操作在这里请见下图。注：字符串是不可改变的。</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-2.png" /></p>
-<p>关于编码这个大坑，在Python3中做的已经比Python2好太多，在这里也不做介绍了。介绍一些关于编码处理的库。</p>
-<p><code>codecs</code>、<code>unicodedata</code></p>
-<p>正则表达式部分，推荐自主学习。难度系数不算高。搜索关键词，Python,
-正则表达式,
-re即可。值得一提的是，<code>re</code>是目前我了解处理文本速度最快最优雅的一种方式。</p>
-<p>英文文本是高度冗余的，忽略掉词内部的元音仍然可以很容易的阅读，有些时候这很明显。例如：declaration
-变成 dclrtn，inalienable 变成 inlnble，保留所有词首或词尾的元音序列
-。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"></span><br><span class="line">regexp = <span class="string">r&#x27;^[AEIOUaeiou]+|[AEIOUaeiou]+$|[^AEIOUaeiou]&#x27;</span></span><br><span class="line"><span class="comment"># 压缩元音字符串</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">compress</span>(<span class="params">word</span>):</span><br><span class="line">	pieces = re.findall(regexp, word)</span><br><span class="line">    <span class="keyword">return</span> <span class="string">&#x27;&#x27;</span>.join(pieces)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 获取语料库数据</span></span><br><span class="line">english_udhr = nltk.corpus.udhr.words(<span class="string">&#x27;English-Latin1&#x27;</span>)</span><br><span class="line"><span class="comment"># 进行压缩</span></span><br><span class="line">nltk.tokenwrap(compress(w) <span class="keyword">for</span> w <span class="keyword">in</span> english_udhr[:<span class="number">75</span>])</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则结合条件频率，提取辅音-元音序列</span></span><br><span class="line">rotokas_words = nltk.corpus.toolbox.words(<span class="string">&#x27;rotokas.dic&#x27;</span>)</span><br><span class="line">cvs = [cv <span class="keyword">for</span> w <span class="keyword">in</span> rotokas_words <span class="keyword">for</span> cv <span class="keyword">in</span> re.findall(<span class="string">r&#x27;[ptksvr][aeiou]&#x27;</span>, w)]</span><br><span class="line">cfd = nltk.ConditionalFreqDist(cvs)</span><br><span class="line">cfd.tabulate()</span><br><span class="line"></span><br><span class="line">    a   e   i   o   u</span><br><span class="line">k <span class="number">418</span> <span class="number">148</span>  <span class="number">94</span> <span class="number">420</span> <span class="number">173</span></span><br><span class="line">p  <span class="number">83</span>  <span class="number">31</span> <span class="number">105</span>  <span class="number">34</span>  <span class="number">51</span></span><br><span class="line">r <span class="number">187</span>  <span class="number">63</span>  <span class="number">84</span>  <span class="number">89</span>  <span class="number">79</span></span><br><span class="line">s   <span class="number">0</span>   <span class="number">0</span> <span class="number">100</span>   <span class="number">2</span>   <span class="number">1</span></span><br><span class="line">t  <span class="number">47</span>   <span class="number">8</span>   <span class="number">0</span> <span class="number">148</span>  <span class="number">37</span></span><br><span class="line">v  <span class="number">93</span>  <span class="number">27</span> <span class="number">105</span>  <span class="number">48</span>  <span class="number">49</span></span><br></pre></td></tr></table></figure>
-<h3 id="词干提取器">词干提取器</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 查找词干，laptop与laptops其实只是单复数的区别。词干是相同的</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">stem</span>(<span class="params">word</span>):</span><br><span class="line">    regexp = <span class="string">r&#x27;^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)?$&#x27;</span></span><br><span class="line">    stem, suffix = re.findall(regexp, word)[<span class="number">0</span>]</span><br><span class="line">    <span class="keyword">return</span> stem</span><br><span class="line"></span><br><span class="line"><span class="comment"># nltk自带的词干提取器</span></span><br><span class="line"><span class="comment"># 索引一些文本和使搜索支持不同词汇形式的话，使用Porter词干提取器</span></span><br><span class="line">porter = nltk.PorterStemmer()</span><br><span class="line">lancaster = nltk.LancasterStemmer()</span><br><span class="line"><span class="comment"># 编译一些文本的词汇或想要一个有效词条（或中心词）列表，使用WordNet词形归并器</span></span><br><span class="line">wnl = nltk.WordNetLemmatizer()</span><br></pre></td></tr></table></figure>
-<h3 id="分词">分词：</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用正则进行分词</span></span><br><span class="line">re.findall(<span class="string">r&quot;\w+(?:[-&#x27;]\w+)|&#x27;|[-.(]+|\S\w&quot;</span>, raw)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用nltk自带的函数进行分词，效率更高且不需要特殊处理符号</span></span><br><span class="line">nltk.regexp_tokenize()</span><br><span class="line"></span><br><span class="line">pattern = <span class="string">r&#x27;&#x27;&#x27;(?x) # 去除嵌入的空白字符和注释</span></span><br><span class="line"><span class="string">			([A-Z]\.)+ # 设定允许的标识符（单词）</span></span><br><span class="line"><span class="string">			| \w+(-\w+)* # 可选的内部连字符</span></span><br><span class="line"><span class="string">			| \$?\d+(\.\d+)?%? # 货币与百分比</span></span><br><span class="line"><span class="string">			| \.\.\. # 省略符</span></span><br><span class="line"><span class="string">			| [][.,;&quot;&#x27;?():-_`] # 额外的标志</span></span><br><span class="line"><span class="string">		&#x27;&#x27;&#x27;</span></span><br><span class="line">nltk.regexp_tokenize(text, pattern)</span><br></pre></td></tr></table></figure>
-<p>已经分词好的数据举例：</p>
-<p>《华尔街日报》原始文本（<code>nltk.corpus.treebank_raw.raw()</code>）和分好词的版本（<code>nltk.corpus.treebank.words()</code>）</p>
-<p>分词的最后一个问题是缩写的存在，如“didn't”。如果我们想分析一个句子的意思，将这种形式规范化为两个独立的形式：“did”和“n't”(不是
-not)可能更加有用。我们可以通过查表来做这项工作。</p>
-<p>链表与字符串部分不做介绍，是Python字符串的常规操作</p>
-<p>补充部分：</p>
+<h2 id="cnn卷积神经网络">CNN（卷积神经网络）</h2>
+<p><a
+href="https://www.zybuluo.com/hanbingtao/note/485480">零基础入门深度学习(4)
+- 卷积神经网络</a></p>
+<p>CNN（Convolutional Neural
+Netword，卷积神经网络）是一种前馈神经网络，人工神经元可以响应周围单元，可以大型图像处理。卷积神经网络包括卷积层和池化层。</p>
+<p>基础结构</p>
 <ul>
-<li><p>建立词汇表流程</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-1.png" /></p></li>
-<li><p>编码点：每个字符分配一个编号</p></li>
-<li><p>解码：通过一些机制来将文本翻译成 Unicode的过程</p></li>
-<li><p>编码：将Unicode转化为其它编码的过程</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/nltk/3-3.png" /></p></li>
-<li><p>从 Unicode的角度来看，字符是可以实现一个或多个
-字形的抽象的实体。只有字形可以出现在屏幕上或被打印在纸上。一个字体是一个字符到字形映射。</p></li>
-<li><p>规范化文本：如将所有的文本小写保存，使用<code>lower()</code>这个函数</p></li>
-<li><p>​断句：在将文本分词之前，我们需要将它分割成句子</p></li>
-<li><p>分词（重点内容）：</p>
-<ol type="1">
-<li><p>找到一种方法来分开文本内容与分词标志。给每个字符标注一个布尔值来指示这个字符后面是否有一个分词标志</p></li>
-<li><p>找到将文本字符串正确分割成词汇的字位串。根据一个字典，来根据字典中词的序列来重构源文本。定义一个目标函数，通过字典大小与重构源文本所需的信息量尽力优化它的值。</p></li>
-<li><p>找到最大化目标函数值的0和1的模式。使用<strong>模拟退火算法</strong>的非确定性搜索。</p></li>
-</ol></li>
+<li>特征提取层，每个神经元的输入与前一层的局部接受域相连，并提取该局部的特征。一旦该局部特征被提取后，它与其他特征间的位置关系也随之确定下来</li>
+<li>特征映射层，网络的每个计算层由多个特征映射组成，每个特征映射是一个平面，平面上所有的神经元的权值相等。采用影响函数较小的<span
+class="math inline">\(sigmoid\)</span>函数作为卷积网络的激活函数，使得特征映射具有位移不变性</li>
 </ul>
-<h2 id="第四章编写结构化程序">第四章：编写结构化程序</h2>
-<p>大量关于Python代码，不做介绍。此部分适宜阅读人群，使用Python时间不超过一年（不够熟练）</p>
-<p>文档说明模块：<code>docstring</code></p>
-<p>调试技术：<code>pdb</code></p>
-<p>绘图：<code>matplotlib</code></p>
-<p>图的绘制：<code>networkx</code></p>
-<p>数据统计表：<code>csv</code>（这里个人推荐有使用<code>pandas</code>，虽然它很”重“）</p>
-<p>数值运算：<code>numpy</code></p>
-<p>算法设计：</p>
+<p>CNN主要用来识别位移、缩放以及其他形式扭曲不变性的二维图形。由于CNN的特征检测层通过训练数据进行学习，所以在使用CNN时，避免了显示的特征抽取，从而隐式地从训练数据中进行学习。由于同一特征映射面上的神经元权值相同，所以网络可以并行学习。</p>
+<p>对于图像识别任务</p>
+<p>全连接网络</p>
 <ol type="1">
-<li>递归</li>
-<li>权衡空间与时间</li>
-<li>动态规划</li>
+<li>参数数量过多</li>
+<li>像素间的位置信息</li>
+<li>网络层数限制</li>
 </ol>
-<h2 id="第五章分类和标注词汇">第五章：分类和标注词汇</h2>
-<p>NLP基本技术：</p>
+<p>解决方案：</p>
+<ol type="1">
+<li>局部连接——不是全连接，减少参数数量</li>
+<li>权值共享——一组连接可以共享共一个权重，再一次减少参数数量</li>
+<li>下采样——使用池化（Pooling）来减少每层的样本数，进一步减少参数数量，提供<strong>鲁棒性
+</strong></li>
+</ol>
+<p><strong>P.S.</strong></p>
+<p>鲁棒是Robust的音译，也就是健壮和强壮的意思。它是在异常和危险情况下系统生存的关键。比如说，计算机软件在输入错误、磁盘故障、网络过载或有意攻击情况下，能否不死机、不崩溃，就是该软件的鲁棒性。所谓“鲁棒性”，是指控制系统在一定（结构，大小）的参数摄动下，维持其它某些性能的特性。根据对性能的不同定义，可分为稳定鲁棒性和性能鲁棒性。以闭环系统的鲁棒性作为目标设计得到的固定控制器称为鲁棒控制器。</p>
+<p>卷积神经网络：<strong>卷积层、Pooling层（池化层）、全连接层</strong></p>
+<p><code>INPUT -&gt; [[CONV]*N -&gt; POOL]*M -&gt; [FC]*K</code></p>
+<p>N个卷积层叠加再加上一个池化层，重复M次，加上K个全连接层。</p>
+<p>概念介绍和计算方式请看下面这篇博客</p>
+<p><a
+href="http://www.cnblogs.com/charlotte77/p/7759802.html">【深度学习系列】卷积神经网络CNN原理详解(一)——基本原理</a></p>
+<p>总结：</p>
+<ol type="1">
+<li>池化方式：
+<ol type="1">
+<li>MaxPooling：取滑动窗口里最大的值</li>
+<li>AveragePooling：取滑动窗口内所有值的平均值</li>
+<li>StochasticPooling，介于两者之间，通过对像素点按照数值大小赋予概率，再按照概率进行亚采样；</li>
+</ol></li>
+<li>Zero Padding(补零)：保留原始图片的尺寸</li>
+</ol>
+<h4 id="卷积">卷积</h4>
+<p>对于一个<span class="math inline">\(l_1*l_2\)</span>的图像，使用<span
+class="math inline">\(f_1*f_2\)</span>的filter进行卷积（后者小于前者），得到一个<span
+class="math inline">\(f_3*f_4\)</span>的feature map。</p>
+<p><span class="math display">\[
+a_{ij}=f(\sum\limits_{m=0}^{f_1-1}\sum\limits_{n=0}^{f_2-1}w_{mn}x_{(m+i)(n+j)}+w_b)
+\]</span></p>
+<p><span class="math inline">\(a_{ij}\)</span>表示的是feature
+map的元素，<span
+class="math inline">\(w_{mn}\)</span>表示对应元素，<span
+class="math inline">\(f\)</span>函数表示激活函数，<span
+class="math inline">\(x_{(m+i)(n+j)}\)</span>表示图像的元素，<span
+class="math inline">\(w_b\)</span>表示偏置项，这里从0开始计数，所以需要-1。</p>
+<p>这里简单起见，步幅设置为1。</p>
+<p>图像大小、步幅和卷积后的Feature Map大小是有关系的。</p>
+<p><span class="math inline">\(W_2=(W_1-F+2P)/S+1\)</span></p>
+<p><span class="math inline">\(H_2=(H_1-F+2P)/S+1\)</span></p>
+<p><span class="math inline">\(W_2\)</span>是卷积后Feature
+Map的宽度；<span
+class="math inline">\(W_1\)</span>是卷积前图像的宽度；<span
+class="math inline">\(F\)</span>是filter的宽度；<span
+class="math inline">\(P\)</span>是<strong>Zero
+Padding</strong>数量，<strong>Zero
+Padding</strong>是指在原始图像周围补几圈0，如果的值是1，那么就补1圈0；<span
+class="math inline">\(S\)</span>是<strong>步幅</strong>；<span
+class="math inline">\(H_2\)</span>是卷积后Feature Map的高度；<span
+class="math inline">\(H_1\)</span>是卷积前图像的宽度。</p>
+<p>二维卷积公式，已知矩阵<span class="math inline">\(A\)</span>、<span
+class="math inline">\(B\)</span>，行列数分别用<span
+class="math inline">\(m\)</span>和<span
+class="math inline">\(n\)</span>表示。卷积公式如下</p>
+<p><span
+class="math inline">\(C_{st}=\sum\limits_0^{m_A-1}\sum\limits_0^{n_A-1}A_{mn}B_{(s-m)(t-n)}\)</span></p>
+<p><span class="math inline">\(0\le s&lt;m_A+m_B-1\)</span></p>
+<p><span class="math inline">\(0\le t&lt;n_A+n_B-1\)</span></p>
+<p>也可以写成<span class="math inline">\(C=A*B\)</span></p>
+<p>数学中的卷积与卷积神经网络中的卷积是一定区别的。可以把神经网络中的卷积称为<strong>互相关(cross-correlation)</strong>。这两种操作转换方式，可以通过把矩阵<span
+class="math inline">\(A\)</span>翻转180度，再交换<span
+class="math inline">\(A\)</span>、<span
+class="math inline">\(B\)</span>运算位置，即可以将卷积变为互相关。</p>
+<h2 id="rnn循环神经网络">RNN（循环神经网络）</h2>
+<p>RNN（Recurrent neural
+Network，循环神经网络/多层反馈网络）是一种节点定向连接成环的人工神经网络。这种网络的内部状态可以展示动态时序行为。不同于前馈神经网络的是，RNN可以利用它内部的记忆来处理任意时序的输入序列，这让它可以更容易处理，如不分段的手写识别、语音识别等。</p>
+<p>RNN网络的本质特征是处理单元之间既有内部的反馈连接又有前馈连接。从系统的观点看，它是一个反馈动力系统，在计算过程中体现过程动态特性，比前馈神经网络具有更强的动态行为和计算能力。</p>
+<p><a
+href="https://iamtrask.github.io/2015/11/15/anyone-can-code-lstm/">参考文章</a></p>
+<p><strong><em>Talk</em> <em>is</em> cheap. <em>Show</em> <em>me</em>
+<em>the</em> <em>code</em>.</strong></p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> copy</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="comment"># sigmoid激活函数</span></span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 存储10进制转2进制的字典</span></span><br><span class="line">int2binary = &#123;&#125;</span><br><span class="line"><span class="comment"># 二进制数最大长度</span></span><br><span class="line">binary_dim = <span class="number">8</span></span><br><span class="line"><span class="comment"># 最大10进制数</span></span><br><span class="line">larget_number = <span class="number">2</span> ** binary_dim</span><br><span class="line">binary = np.unpackbits(</span><br><span class="line">    np.array([<span class="built_in">range</span>(larget_number)], dtype=np.uint8).T, axis=<span class="number">1</span>)</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(larget_number):</span><br><span class="line">    int2binary[i] = binary[i]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 梯度</span></span><br><span class="line">alpha = <span class="number">0.1</span></span><br><span class="line"><span class="comment"># 输入维度，两个数</span></span><br><span class="line">input_dim = <span class="number">2</span></span><br><span class="line"><span class="comment"># 隐单元维度</span></span><br><span class="line">hidden_dim = <span class="number">16</span></span><br><span class="line"><span class="comment"># 输出维度，一个数</span></span><br><span class="line">output_dim = <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 两层神经网络。每层权重，[-1, 1]</span></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((input_dim, hidden_dim)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((hidden_dim, output_dim)) - <span class="number">1</span></span><br><span class="line"><span class="comment"># 神经网络连接前一个隐藏层的权重</span></span><br><span class="line">syn_h = <span class="number">2</span> * np.random.random((hidden_dim, hidden_dim)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 存储权重的更新</span></span><br><span class="line">syn_0_update = np.zeros_like(syn_0)</span><br><span class="line">syn_1_update = np.zeros_like(syn_1)</span><br><span class="line">syn_h_update = np.zeros_like(syn_h)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    <span class="comment"># 一个简单的加法问题，c=a+b</span></span><br><span class="line">    <span class="comment"># 在0，128（避免相加超过256）之间一个随机数a，十进制表示</span></span><br><span class="line">    a_int = np.random.randint(larget_number / <span class="number">2</span>)</span><br><span class="line">    <span class="comment"># 二进制表示a</span></span><br><span class="line">    a = int2binary[a_int]</span><br><span class="line"></span><br><span class="line">    b_int = np.random.randint(larget_number / <span class="number">2</span>)</span><br><span class="line">    b = int2binary[b_int]</span><br><span class="line">    <span class="comment"># 正确答案c</span></span><br><span class="line">    c_int = a_int + b_int</span><br><span class="line">    c = int2binary[c_int]</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 神经网络计算的答案</span></span><br><span class="line">    d = np.zeros_like(c)</span><br><span class="line">    <span class="comment"># 计算的误差</span></span><br><span class="line">    overallError = <span class="number">0</span></span><br><span class="line">    <span class="comment"># 记录l2_delta、l1</span></span><br><span class="line">    layer_2_deltas = []</span><br><span class="line">    layer_1_values = []</span><br><span class="line">    <span class="comment"># l1初始值全为0</span></span><br><span class="line">    layer_1_values.append(np.zeros(hidden_dim))</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 前向传播。特征是二进制数维度。</span></span><br><span class="line">    <span class="keyword">for</span> position <span class="keyword">in</span> <span class="built_in">range</span>(binary_dim):</span><br><span class="line">        <span class="comment"># X是1*2的矩阵，[[m, n]]，m、n是二进制数a、b在每个位置的01值。从最后一维开始遍历</span></span><br><span class="line">        X = np.array(</span><br><span class="line">            [[a[binary_dim - position - <span class="number">1</span>], b[binary_dim - position - <span class="number">1</span>]]])</span><br><span class="line">        <span class="comment"># y是1*1的矩阵。[[t]]，t是二进制数c在每个位置的01值</span></span><br><span class="line">        y = np.array([[c[binary_dim - position - <span class="number">1</span>]]]).T</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 计算输出层，np.dot(X,syn_0)是正常操作:从输入层到隐藏层;np.dot(layer_1_values[-1],syn_h):从前一层隐藏层到这一层隐藏层</span></span><br><span class="line">        layer_1 = sigmoid(np.dot(X, syn_0) + np.dot(layer_1_values[-<span class="number">1</span>], syn_h))</span><br><span class="line">        layer_2 = sigmoid(np.dot(layer_1, syn_1))</span><br><span class="line">        <span class="comment"># 计算l2偏差</span></span><br><span class="line">        layer_2_error = y - layer_2</span><br><span class="line">        <span class="comment"># 保存偏差加权导数l2_delta</span></span><br><span class="line">        layer_2_deltas.append(layer_2_error * sigmoid(layer_2, deriv=<span class="literal">True</span>))</span><br><span class="line">        <span class="comment"># 误差总和</span></span><br><span class="line">        overallError += np.<span class="built_in">abs</span>(layer_2_error[<span class="number">0</span>])</span><br><span class="line">        <span class="comment"># 神经网络每个维度的计算的01值</span></span><br><span class="line">        d[binary_dim - position - <span class="number">1</span>] = np.<span class="built_in">round</span>(layer_2[<span class="number">0</span>][<span class="number">0</span>])</span><br><span class="line">        <span class="comment"># 保存l1层，在下次迭代时计算下次传播的隐藏层</span></span><br><span class="line">        <span class="comment"># layer_1_values.append(copy.deepcopy(layer_1))</span></span><br><span class="line">        layer_1_values.append(layer_1.copy())</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 存储反向传播的l1_delta</span></span><br><span class="line">    futuer_layer_1_delta = np.zeros(hidden_dim)</span><br><span class="line">    <span class="comment"># 反向传播.(input + prev_input) -&gt; hidden -&gt; output</span></span><br><span class="line">    <span class="keyword">for</span> position <span class="keyword">in</span> <span class="built_in">range</span>(binary_dim):</span><br><span class="line">        <span class="comment"># 从第一维开始遍历</span></span><br><span class="line">        X = np.array([[a[position], b[position]]])</span><br><span class="line">        <span class="comment"># 选出前面计算得到的l1作为当前层，反向计算</span></span><br><span class="line">        layer_1 = layer_1_values[-position - <span class="number">1</span>]</span><br><span class="line">        <span class="comment"># 选择l1前一层的隐藏层</span></span><br><span class="line">        prev_layer_1 = layer_1_values[-position - <span class="number">2</span>]</span><br><span class="line">        <span class="comment"># 选择出l2，当前层输出的误差</span></span><br><span class="line">        layer_2_delta = layer_2_deltas[-position - <span class="number">1</span>]</span><br><span class="line">        <span class="comment"># 计算当前隐藏层误差，计算前一层的l1_delta到当前层，加上正常操作。</span></span><br><span class="line">        layer_1_delta = (np.dot(futuer_layer_1_delta, syn_h.T) +</span><br><span class="line">                         layer_2_delta.dot(syn_1.T)) * sigmoid(layer_1, deriv=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 更新权重更新的参数。将l1转换为二维矩阵</span></span><br><span class="line">        syn_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)</span><br><span class="line">        syn_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)</span><br><span class="line">        <span class="comment"># X已经是二维矩阵，不需要转换</span></span><br><span class="line">        syn_0_update += X.T.dot(layer_1_delta)</span><br><span class="line">        <span class="comment"># 替换futuer_l1</span></span><br><span class="line">        futuer_layer_1_delta = layer_1_delta</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 调整每层权重</span></span><br><span class="line">    syn_0 += syn_0_update * alpha</span><br><span class="line">    syn_1 += syn_1_update * alpha</span><br><span class="line">    syn_h += syn_h_update * alpha</span><br><span class="line">    <span class="comment"># 归零更新权重参数</span></span><br><span class="line">    syn_0_update *= <span class="number">0</span></span><br><span class="line">    syn_1_update *= <span class="number">0</span></span><br><span class="line">    syn_h_update *= <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (_ % <span class="number">1000</span> == <span class="number">0</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Error: &quot;</span> + <span class="built_in">str</span>(overallError))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Pred: &quot;</span> + <span class="built_in">str</span>(d))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;True: &quot;</span> + <span class="built_in">str</span>(c))</span><br><span class="line">        <span class="comment"># 二进制转十进制</span></span><br><span class="line">        out = <span class="number">0</span></span><br><span class="line">        <span class="keyword">for</span> index, x <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">reversed</span>(d)):</span><br><span class="line">            out += x * (<span class="number">2</span>**index)</span><br><span class="line">        <span class="built_in">print</span>(<span class="built_in">str</span>(a_int) + <span class="string">&quot; + &quot;</span> + <span class="built_in">str</span>(b_int) + <span class="string">&quot;=&quot;</span> + <span class="built_in">str</span>(out))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;-------------&quot;</span>)</span><br></pre></td></tr></table></figure>
+<p>将前向传播比喻为字母表顺序读出，反向传播就是字母表反序读出。</p>
+<h2 id="lstm时间递归神经网络">LSTM（时间递归神经网络）</h2>
+<p>长短期记忆（英语：Long Short-Term
+Memory，LSTM）是一种时间递归神经网络（RNN），论文首次发表于1997年。由于独特的设计结构，LSTM适合于处理和预测时间序列中间隔和延迟非常长的重要事件。</p>
+<p>LSTM的表现通常比时间递归神经网络及隐马尔科夫模型（HMM）更好，比如用在不分段连续手写识别上。作为非线性模型，LSTM可作为复杂的非线性单元用于构造更大型深度神经网络。</p>
+]]></content>
+      <categories>
+        <category>DeepLearning</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>神经网络</tag>
+        <tag>NumPy</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>拉格朗日乘子法：从函数的求导到不等式约束的求解</title>
+    <url>/hexo_blog/2018/04/30/%E6%95%B0%E5%AD%A6/%E6%8B%89%E6%A0%BC%E6%9C%97%E6%97%A5%E4%B9%98%E5%AD%90%E6%B3%95%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/</url>
+    <content><![CDATA[<p>本篇文章简单梳理了关于拉格朗日乘子法的相关问题及一些相关概念的介绍。</p>
+<span id="more"></span>
+<h3 id="基本概念">基本概念</h3>
+<p>拉格朗日乘子法（Lagrange
+Multiplier）在某些地方又叫做拉格朗日乘数法。在求解最优化问题中，这个方法是最常用的方法之一。最优化问题往往抽象为某个函数，即求函数的最大值或最小值。求解最大值和最小值的求解可以相互转换（对偶）。对于求解最优化问题分为以下三种情况（难度依此递增）：</p>
+<h3 id="无约束条件">无约束条件</h3>
+<p>一般情况下，只需要对给出的函数进行求导，令其导函数为0的点可能就是极值点，将可能的结果分别代回原函数进行验证即可。</p>
+<p>假设有这样一个函数<span class="math inline">\(f(x)\)</span>，令<span
+class="math inline">\(g(x)=f^{&#39;}(x)\)</span>，<span
+class="math inline">\(g(x)=0\)</span>的解有<span
+class="math inline">\(x_1,x_2,x_3,\cdots\)</span>。求函数<span
+class="math inline">\(f(x)\)</span>的最优解就只需要分别将<span
+class="math inline">\(x_1,x_2,x_3,\cdots\)</span>代入<span
+class="math inline">\(f(x)\)</span>，比较大小，得到最优解<span
+class="math inline">\(x_i\)</span>。</p>
+<p>从这里开始引出<strong>拉格朗日乘子法</strong></p>
+<h3 id="等式约束条件">等式约束条件</h3>
+<p>设目标函数为<span class="math inline">\(f(x)\)</span>,
+约束条件为$h_k(x) $, <span
+class="math inline">\(k=1,2,\cdots,l\)</span>，即要求<span
+class="math inline">\(f(x)\)</span>的最优解，但是有<span
+class="math inline">\(l\)</span>个<span
+class="math inline">\(h(x)\)</span>等式约束（限制）求解。这里我们定义拉格朗日函数为</p>
+<p><span class="math display">\[
+F(x, \lambda)=f(x)+\sum\limits_{k=1}^{l}\lambda_kh_k(x)
+\]</span></p>
+<p><span
+class="math inline">\(\lambda_k\)</span>表示约束条件的待定系数。之后解变量的偏导方程，如下:</p>
+<p><span class="math display">\[
+\frac{\partial F}{\partial x_k}=0, \frac{\partial F}{\partial
+\lambda_k}=0, k=1,2,\cdots, l
+\]</span></p>
+<p>求解得到了<span
+class="math inline">\(l+1\)</span>个方程，将结果代回原方程验证即可得到解。</p>
+<p>举例：</p>
+<p>给定一个椭圆<span
+class="math inline">\(\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}=1\)</span>，求解<span
+class="math inline">\(f(x,y,z)=8xyz\)</span>的最大值。</p>
+<p>求解过程如下：</p>
 <ul>
-<li>序列标注</li>
-<li>N-gram 模型</li>
-<li>回退和评估</li>
+<li><p>通过拉格朗日乘子法的定义，得到如下方程</p>
+<p><span class="math display">\[
+\begin{array}{l}F(x,y,z,\lambda)&amp;=f(x,y,z)+\lambda\cdot h(x,y,z) \\
+&amp;=8xyz+\lambda\cdot(\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}-1)
+\end{array}
+\]</span></p></li>
+<li><p>接下来对<span
+class="math inline">\(F(x,y,z,\lambda)\)</span>求偏导，分别得到</p>
+<p><span class="math display">\[
+\frac{\partial F(x,y,z,\lambda)}{\partial x}=8yz+\frac{2\lambda
+x}{a^2}=0
+\]</span></p>
+<p><span class="math display">\[
+\frac{\partial F(x,y,z,\lambda)}{\partial y}=8xz+\frac{2\lambda
+y}{b^2}=0
+\]</span></p>
+<p><span class="math display">\[
+\frac{\partial F(x,y,z,\lambda)}{\partial z}=8xy+\frac{2\lambda
+z}{c^2}=0
+\]</span></p>
+<p><span class="math display">\[
+\frac{\partial F(x,y,z,\lambda)}{\partial
+\lambda}=\frac{x^2}{a^2}+\frac{y^2}{b^2}+\frac{z^2}{c^2}-1=0
+\]</span></p></li>
+<li><p>联立前三个方程可以得到<span
+class="math inline">\(bx=ay\)</span>和<span
+class="math inline">\(az=cx\)</span>，代入最后一个方程，解得：</p>
+<p><span class="math display">\[
+x=\frac{\sqrt{3}}{3}a
+\]</span></p>
+<p><span class="math display">\[
+y=\frac{\sqrt{3}}{3}b
+\]</span></p>
+<p><span class="math display">\[
+z=\frac{\sqrt{3}}{3}c
+\]</span></p></li>
+<li><p>最后代入原方程，解得最大体积</p>
+<p><span class="math display">\[
+V_{max}=f(\frac{\sqrt{3}}{3}a,\frac{\sqrt{3}}{3}b,\frac{\sqrt{3}}{3}c)=\frac{8\sqrt{3}}{9}abc
+\]</span></p></li>
 </ul>
-<p>将词汇按它们的<strong>词性</strong>（parts-of-speech，POS）分类以及相应的标注它们的过程被称为<strong>词性标注</strong>（part-of-speech
-tagging, POS
-tagging）或干脆简称<strong>标注</strong>。词性也称为<strong>词类</strong>或<strong>词汇范畴</strong>。用于特定任务的标记的集合被称为一个<strong>标记集</strong>。</p>
-<h3 id="词性标注器">词性标注器</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line">text = nltk.word_tokenize(<span class="string">&quot;And now for something completely different&quot;</span>)</span><br><span class="line"><span class="comment"># 标注出text中每个词的词性</span></span><br><span class="line">nltk.pos_tag(text)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">[(&#x27;And&#x27;, &#x27;CC&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;now&#x27;, &#x27;RB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;for&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;something&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;completely&#x27;, &#x27;RB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;different&#x27;, &#x27;JJ&#x27;)]</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line">text = nltk.Text(word.lower() <span class="keyword">for</span> word <span class="keyword">in</span> nltk.corpus.brown.words())</span><br><span class="line"><span class="comment"># 为一个词找出所有它的上下文</span></span><br><span class="line">text.similar(<span class="string">&#x27;woman&#x27;</span>)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">man time day year car moment world house family child country boy</span></span><br><span class="line"><span class="string">state job place way war girl work word</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br></pre></td></tr></table></figure>
-<h3 id="标注语料库">标注语料库</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 创建一个有标识的字符串</span></span><br><span class="line">tagged_token = nltk.tag.str2tuple(<span class="string">&#x27;fly/NN&#x27;</span>)</span><br><span class="line">tagged_token</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">tagged_token</span></span><br><span class="line"><span class="string">(&#x27;fly&#x27;, &#x27;NN&#x27;)</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line">tagged_token[<span class="number">0</span>]</span><br><span class="line">tagged_token[<span class="number">1</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 批量标注</span></span><br><span class="line">sent = <span class="string">&#x27;&#x27;&#x27;The/AT grand/JJ jury/NN commented/VBD on/IN a/AT number/NN of/INother/AP topics/NNS ,/, AMONG/IN them/PPO the/AT Atlanta/NP and/CCFulton/NP-tl County/NN-tl purchasing/VBG departments/NNS which/WDT it/PPsaid/VBD ``/`` ARE/BER well/QL operated/VBN and/CC follow/VB generally/Raccepted/VBN practices/NNS which/WDT inure/VB to/IN the/AT best/JJTinterest/NN of/IN both/ABX governments/NNS &#x27;&#x27;/&#x27;&#x27; ./.&#x27;&#x27;&#x27;</span></span><br><span class="line">[nltk.tag.str2tuple(t) <span class="keyword">for</span> t <span class="keyword">in</span> sent.split()]</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">[(&#x27;The&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;grand&#x27;, &#x27;JJ&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;jury&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;commented&#x27;, &#x27;VBD&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;on&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;a&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;number&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;of/INother&#x27;, &#x27;AP&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;topics&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;,&#x27;, &#x27;,&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;AMONG&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;them&#x27;, &#x27;PPO&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;the&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;Atlanta&#x27;, &#x27;NP&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;and/CCFulton&#x27;, &#x27;NP-TL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;County&#x27;, &#x27;NN-TL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;purchasing&#x27;, &#x27;VBG&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;departments&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;which&#x27;, &#x27;WDT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;it/PPsaid&#x27;, &#x27;VBD&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;``&#x27;, &#x27;``&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;ARE&#x27;, &#x27;BER&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;well&#x27;, &#x27;QL&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;operated&#x27;, &#x27;VBN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;and&#x27;, &#x27;CC&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;follow&#x27;, &#x27;VB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;generally/Raccepted&#x27;, &#x27;VBN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;practices&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;which&#x27;, &#x27;WDT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;inure&#x27;, &#x27;VB&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;to&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;the&#x27;, &#x27;AT&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;best/JJTinterest&#x27;, &#x27;NN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;of&#x27;, &#x27;IN&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;both&#x27;, &#x27;ABX&#x27;),</span></span><br><span class="line"><span class="string"> (&#x27;governments&#x27;, &#x27;NNS&#x27;),</span></span><br><span class="line"><span class="string"> (&quot;&#x27;&#x27;&quot;, &quot;&#x27;&#x27;&quot;),</span></span><br><span class="line"><span class="string"> (&#x27;.&#x27;, &#x27;.&#x27;)]</span></span><br><span class="line"><span class="string"> &#x27;&#x27;&#x27;</span></span><br></pre></td></tr></table></figure>
-<h3 id="获取已标注的语料库">获取已标注的语料库</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 获取标注好的语料库</span></span><br><span class="line">nltk.corpus.brown.tagged_words()</span><br><span class="line"><span class="comment">#</span></span><br><span class="line">nltk.corpus.brown.tagged_words(tagset=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">nltk.corpus.nps_chat.tagged_words()</span><br><span class="line">nltk.corpus.conll2000.tagged_words()</span><br></pre></td></tr></table></figure>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 未简化的标记</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 最频繁的名词标记</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">findtags</span>(<span class="params">tag_prefix, tagged_text</span>):</span><br><span class="line">	cfd = nltk.ConditionalFreqDist((tag, word) <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> tagged_text <span class="keyword">if</span> tag.startswith(tag_prefix))</span><br><span class="line">    <span class="keyword">return</span> <span class="built_in">dict</span>((tag, cfd[tag].keys()[:<span class="number">5</span>]) <span class="keyword">for</span> tag <span class="keyword">in</span> cfd.conditions())</span><br><span class="line"></span><br><span class="line">tagdict = findtags(<span class="string">&#x27;NN&#x27;</span>, nltk.corpus.brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line"><span class="keyword">for</span> tag <span class="keyword">in</span> <span class="built_in">sorted</span>(tagdict):</span><br><span class="line">	<span class="built_in">print</span> tag, tagdict[tag]</span><br><span class="line"></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">$：名词所有格,</span></span><br><span class="line"><span class="string">S：复数名词（因为复数名词通常以 s结尾），</span></span><br><span class="line"><span class="string">P：专有名词</span></span><br><span class="line"><span class="string">-NC：引用</span></span><br><span class="line"><span class="string">-HL：标题中的词</span></span><br><span class="line"><span class="string">-TL：标题（布朗标记的特征）</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 探索已标注的语料库</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用POS标记寻找三词短语</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="keyword">def</span> <span class="title function_">process</span>(<span class="params">sentence</span>):</span><br><span class="line">	<span class="keyword">for</span> (w1,t1), (w2,t2), (w3,t3) <span class="keyword">in</span> nltk.trigrams(sentence):</span><br><span class="line">         <span class="comment"># 动词+TO+动词</span></span><br><span class="line">		<span class="keyword">if</span> (t1.startswith(<span class="string">&#x27;V&#x27;</span>) <span class="keyword">and</span> t2 == <span class="string">&#x27;TO&#x27;</span> <span class="keyword">and</span> t3.startswith(<span class="string">&#x27;V&#x27;</span>)):</span><br><span class="line">			<span class="built_in">print</span>(w1, w2, w3)</span><br><span class="line"><span class="keyword">for</span> tagged_sent <span class="keyword">in</span> brown.tagged_sents():</span><br><span class="line">	process(tagged_sent)</span><br></pre></td></tr></table></figure>
-<p>Python字典部分不做介绍。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 使用默认字典可以防止使用未定义的key报错</span></span><br><span class="line"><span class="comment"># nltk的默认字典,需要提前定义类型（int、float、str、list、dict、tuple）。</span></span><br><span class="line">frequency = nltk.defaultdict(<span class="built_in">int</span>)</span><br><span class="line"><span class="comment"># 创建具有默认值的字典</span></span><br><span class="line">pos = nltk.defaultdict(<span class="keyword">lambda</span>: <span class="string">&#x27;N&#x27;</span>)</span><br></pre></td></tr></table></figure>
-<h3 id="自动标注">自动标注</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"></span><br><span class="line">brown_tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">brown_sents = brown.sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 默认标注器</span></span><br><span class="line">tags = [tag <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>)]</span><br><span class="line">nltk.FreqDist(tags).<span class="built_in">max</span>()</span><br><span class="line"><span class="comment"># 将所有词都标注为NN的标注器</span></span><br><span class="line">raw = <span class="string">&#x27;I do not like green eggs and ham, I do not like them Sam I am!&#x27;</span></span><br><span class="line">tokens = nltk.word_tokenize(raw)</span><br><span class="line">default_tagger = nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>)</span><br><span class="line">default_tagger.tag(tokens)</span><br><span class="line"><span class="comment"># 评判标注正确性</span></span><br><span class="line">default_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 正则标注</span></span><br><span class="line">patterns = [</span><br><span class="line">	(<span class="string">r&#x27;.*ing$&#x27;</span>, <span class="string">&#x27;VBG&#x27;</span>), <span class="comment"># gerunds</span></span><br><span class="line">	(<span class="string">r&#x27;.*ed$&#x27;</span>, <span class="string">&#x27;VBD&#x27;</span>), <span class="comment"># simple past</span></span><br><span class="line">	(<span class="string">r&#x27;.*es$&#x27;</span>, <span class="string">&#x27;VBZ&#x27;</span>), <span class="comment"># 3rd singular present</span></span><br><span class="line">	(<span class="string">r&#x27;.*ould$&#x27;</span>, <span class="string">&#x27;MD&#x27;</span>), <span class="comment"># modals</span></span><br><span class="line">	(<span class="string">r&#x27;.*\&#x27;s$&#x27;</span>, <span class="string">&#x27;NN$&#x27;</span>), <span class="comment"># possessive nouns</span></span><br><span class="line">	(<span class="string">r&#x27;.*s$&#x27;</span>, <span class="string">&#x27;NNS&#x27;</span>), <span class="comment"># plural nouns</span></span><br><span class="line">	(<span class="string">r&#x27;^-?[0-9]+(.[0-9]+)?$&#x27;</span>, <span class="string">&#x27;CD&#x27;</span>), <span class="comment"># cardinal numbers</span></span><br><span class="line">	(<span class="string">r&#x27;.*&#x27;</span>, <span class="string">&#x27;NN&#x27;</span>) <span class="comment"># nouns (default)</span></span><br><span class="line">]</span><br><span class="line">regexp_tagger = nltk.RegexpTagger(patterns)</span><br><span class="line">regexp_tagger.tag(brown_sents[<span class="number">3</span>])</span><br><span class="line">regexp_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 查询标注器</span></span><br><span class="line"><span class="comment"># 找出100个最频繁的词</span></span><br><span class="line">fd = nltk.FreqDist(brown.words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line">cfd = nltk.ConditionalFreqDist(brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>))</span><br><span class="line">most_freq_words = fd.keys()[:<span class="number">100</span>]</span><br><span class="line">likely_tags = <span class="built_in">dict</span>((word, cfd[word].<span class="built_in">max</span>()) <span class="keyword">for</span> word <span class="keyword">in</span> most_freq_words)</span><br><span class="line"><span class="comment"># 使用上面这个信息作为&quot;查找标注器&quot;</span></span><br><span class="line">baseline_tagger = nltk.UnigramTagger(model=likely_tags)</span><br><span class="line">baseline_tagger.evaluate(brown_tagged_sents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 使用信息作为标注器，如果没有找到就使用默认标注器。这个过程称为&quot;回退&quot;</span></span><br><span class="line">baseline_tagger = nltk.UnigramTagger(model=likely_tags, backoff=nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>))</span><br></pre></td></tr></table></figure>
-<h3 id="评估">评估</h3>
-<p>使用<strong>黄金标准</strong>测试数据。一个已经手动标注并作为自动系统评估标准而被接收的语料库。</p>
-<p>如果标注器对词的标记与黄金标准标记相同，那么标注器就被认为是正确的。当然这只是相对于黄金标准这个测试数据而言。</p>
-<p>关于开发一个已标注的语料库，这是一个庞大的任务，其中涉及到了许多方面。</p>
-<p>可以通过<code>nltk.app.concordance()</code>来可视化查找某个语料库中某个单词的词性</p>
-<table>
-<thead>
-<tr>
-<th>标记</th>
-<th>含义</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td>ADJ</td>
-<td>形容词</td>
-</tr>
-<tr>
-<td>ADV</td>
-<td>动词</td>
-</tr>
-<tr>
-<td>CNJ</td>
-<td>连词</td>
-</tr>
-<tr>
-<td>CC</td>
-<td>并列连词</td>
-</tr>
-<tr>
-<td>DET</td>
-<td>限定词</td>
-</tr>
-<tr>
-<td>EX</td>
-<td>存在量词</td>
-</tr>
-<tr>
-<td>FW</td>
-<td>外来词</td>
-</tr>
-<tr>
-<td>MOD</td>
-<td>情态动词</td>
-</tr>
-<tr>
-<td>RB</td>
-<td>副词</td>
-</tr>
-<tr>
-<td>IN</td>
-<td>介词</td>
-</tr>
-<tr>
-<td>N</td>
-<td>名词</td>
-</tr>
-<tr>
-<td>NP</td>
-<td>专有名词</td>
-</tr>
-<tr>
-<td>NUM</td>
-<td>数词</td>
-</tr>
-<tr>
-<td>PRO</td>
-<td>代词</td>
-</tr>
-<tr>
-<td>P</td>
-<td>介词</td>
-</tr>
-<tr>
-<td>TO</td>
-<td>词to</td>
-</tr>
-<tr>
-<td>UH</td>
-<td>感叹词</td>
-</tr>
-<tr>
-<td>V</td>
-<td>动词</td>
-</tr>
-<tr>
-<td>VD</td>
-<td>过去式</td>
-</tr>
-<tr>
-<td>VG</td>
-<td>现在分词</td>
-</tr>
-<tr>
-<td>VN</td>
-<td>过去分词</td>
-</tr>
-<tr>
-<td>WH</td>
-<td>Wh限定词</td>
-</tr>
-<tr>
-<td>NN</td>
-<td>名词</td>
-</tr>
-<tr>
-<td>JJ</td>
-<td>形容词</td>
-</tr>
-<tr>
-<td>VBP</td>
-<td>一般现在时动词</td>
-</tr>
-</tbody>
-</table>
-<h3 id="n-gram标注">N-gram标注</h3>
-<p>1-gram标注器：一元标注器（unigram
-tagger）。用于标注<strong>一个</strong>标识符的上下文的只是标识符本身。</p>
-<p>2-gram 标注器：二元标注器（bigram taggers）</p>
-<p>3-gram 标注器：三元标注器（trigram taggers）</p>
-<p>N-gram标注不考虑句子边界的上下文。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line">brown_tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">brown_sents = brown.sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"><span class="comment"># 标注</span></span><br><span class="line">unigram_tagger = nltk.UnigramTagger(brown_tagged_sents)</span><br><span class="line">bigram_tagger = nltk.BigramTagger(brown_tagged_sents)</span><br><span class="line">trigram_tagger = nltk.TrigramTagger(brown_tagged_sents)</span><br><span class="line"><span class="comment"># 组合标注</span></span><br><span class="line">t0 = nltk.DefaultTagger(<span class="string">&#x27;NN&#x27;</span>)</span><br><span class="line">t1 = nltk.UnigramTagger(train_sents, backoff=t0)</span><br><span class="line">t2 = nltk.BigramTagger(train_sents, backoff=t1)</span><br></pre></td></tr></table></figure>
-<p>存储可以使用python自带的存储模块<code>pickle</code>。</p>
-<p>效果：</p>
+<h3 id="不等式约束条件">不等式约束条件</h3>
+<p>设目标函数为<span class="math inline">\(f(x)\)</span>,
+不等式约束条件为<span class="math inline">\(c_i(x)\)</span>,<span
+class="math inline">\(i=1,2,\cdots,k\)</span>，等式约束条件为$h_j(x) $,
+<span class="math inline">\(j=1,2,\cdots,l\)</span>，即要求<span
+class="math inline">\(f(x)\)</span>的最优解，但是有<span
+class="math inline">\(k\)</span>个<span
+class="math inline">\(g(x)\)</span>不等式和<span
+class="math inline">\(l\)</span>个<span
+class="math inline">\(h(x)\)</span>等式约束（限制）求解。这里我们定义拉格朗日函数为</p>
+<p><span class="math display">\[
+L(x,a,b)=f(x)+\sum\limits^{k}_{i=1}a_ic_i(x)+\sum\limits^{l}_{j=1}b_jh_j(x)
+\]</span></p>
+<p>这里用一个叫做KKT条件的方法，把所有的不等式约束、等式约束和目标函数全部写为一个式子<span
+class="math inline">\(L(a,b,x)=f(x)+a*c(x)+b*h(x)\)</span>。</p>
+<p>KKT条件是指最优解必须满足以下条件：</p>
+<ul>
+<li><span class="math inline">\(L(a,b,x)\)</span>求<span
+class="math inline">\(x\)</span>求导为0</li>
+<li><span class="math inline">\(h(x)=0\)</span></li>
+<li><span class="math inline">\(a*c(x)=0\)</span></li>
+</ul>
+<p>这样就可以将不等式约束转换为了等式约束，求<span
+class="math inline">\(L(a,b,x)\)</span>的偏导等于零即可求得最优参数。</p>
+<p><span class="math display">\[
+\min f(x)\Leftrightarrow \min\limits_x \max\limits_{a,b} L(x,a,b)
+\]</span></p>
+<p>经过对偶变换得：</p>
+<p><span class="math display">\[
+\max\limits_{a,b} \min L(x,a,b)
+\]</span></p>
+<p>因为<span class="math inline">\(c(x)&lt;0\)</span>，所以只有当<span
+class="math inline">\(a*c(x)=0\)</span>时，<span
+class="math inline">\(L(x,a,b)\)</span>才能取得最大值。以上就可以使用同等式约束的求解方法进行求解。</p>
+<h3 id="补充部分">补充部分</h3>
+<h4 id="定义不等式约束问题的标准写法">定义不等式约束问题的标准写法</h4>
+<p><span class="math inline">\(f(x), c_i(x),h_j(x)\)</span>是定义在<span
+class="math inline">\(R^n\)</span>上的连续可微函数，考虑约束最优化问题：</p>
+<p><span class="math inline">\(\min f(x)  \quad  x\in R^n\)</span></p>
+<p><span class="math inline">\(s.t. c_i(x) \leq 0 \quad i =
+1...k\)</span></p>
+<pre><code>$h_j(x)=0 \quad j = 1...l$</code></pre>
+<p>即有<span class="math inline">\(k\)</span>个不等式约束<span
+class="math inline">\(c_i(x)\)</span>和<span
+class="math inline">\(l\)</span>个等式约束<span
+class="math inline">\(h_j(x)\)</span></p>
+<h4 id="拉格朗日函数">拉格朗日函数</h4>
+<p><span class="math inline">\(L(x,\alpha,
+\beta)=f(x)+\sum^{k}_{i=1}\alpha_ic_i(x)+\sum^{l}_{j=1}\beta_jh_j(x)\)</span></p>
+<p>其中的<span class="math inline">\(\alpha_i\)</span>和<span
+class="math inline">\(\beta_j\)</span>，称为<strong>拉格朗日乘子</strong>，<span
+class="math inline">\(\alpha_i \geq 0\)</span></p>
+<h4 id="几何意义">几何意义</h4>
+<p>对于一个二维空间来说，求解最短路径的问题。将维度升高，变为对于多维空间来说，求解最短路径问题。</p>
+<p>比较直观的解释，爬山的目标是到山顶，根据梯度上升（下降）的方法来寻求优解，即爬到山顶。</p>
+<h4 id="对偶概念的简单介绍">对偶概念的简单介绍</h4>
+<p>一对一的一种方式，把一种概念、公理或数学结构转换为另一种概念、公理或数学结构。如果A的对偶是B，那么B的对偶是A。A的对偶有时候是A的本身。</p>
+<p>有界线性泛函数：复杂对象映射到<strong>简单对象</strong>上</p>
+<p>群表示：简单对象具有类似线性结构的<strong>简单结构</strong></p>
+<p>范畴论：映射箭头所构成的集合</p>
+<p>（像集，映射）——&gt;原像集</p>
+<p>对联可以称为一种对偶，上联与下联有对应的关系（相似和取逆）。对偶是有一定要有自然合理的对应关系（或者说能够吹出来的自然合理关系）。</p>
+<p>常见的自然、合理法则首推：相似（包括同构）和取逆（包括互反）</p>
+<p>常见对偶的例子：</p>
 <ol type="1">
-<li><p>根据经验来进行判断——一般方法</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">cfd = nltk.ConditionalFreqDist(((x[<span class="number">1</span>], y[<span class="number">1</span>], z[<span class="number">0</span>]), z[<span class="number">1</span>]) <span class="keyword">for</span> sent <span class="keyword">in</span> brown_tagged_sents <span class="keyword">for</span> x, y, z <span class="keyword">in</span> nltk.trigrams(sent))</span><br><span class="line">ambiguous_contexts = [c <span class="keyword">for</span> c <span class="keyword">in</span> cfd.conditions() <span class="keyword">if</span> <span class="built_in">len</span>(cfd[c]) &gt; <span class="number">1</span>]</span><br><span class="line"><span class="built_in">sum</span>(cfd[c].N() <span class="keyword">for</span> c <span class="keyword">in</span> ambiguous_contexts) / cfd.N()</span><br></pre></td></tr></table></figure></li>
-<li><p>研究标注器的错误——混淆矩阵</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">test_tags = [tag <span class="keyword">for</span> sent <span class="keyword">in</span> brown.sents(categories=<span class="string">&#x27;editorial&#x27;</span>) <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> t2.tag(sent)]</span><br><span class="line">gold_tags = [tag <span class="keyword">for</span> (word, tag) <span class="keyword">in</span> brown.tagged_words(categories=<span class="string">&#x27;editorial&#x27;</span>)]</span><br><span class="line"><span class="built_in">print</span> nltk.ConfusionMatrix(gold, test)</span><br></pre></td></tr></table></figure></li>
+<li>奇数和偶数</li>
+<li>洛必达法则和Stolz 定理（相当于离散洛必达法则）</li>
+<li>Fourier 系数与 Fourier 变换（傅里叶）</li>
 </ol>
-<p>n-gram标注器的问题</p>
+<p>对偶可以理解为，一个问题不好理解转换为另一个简单方式的问题来理解，二者在本质上是相同的。</p>
+<p>比如：某个函数的最大化不方便求解，那么就求这个函数的最小化。</p>
+<p>未完待续……</p>
 <ol type="1">
-<li>模型大小与标注器性能之前的平衡关系。如果使用回退标注器<code>n-gram</code>可能存储trigram和bigram表，这将会是很大的稀疏矩阵。</li>
-<li>使用上下文中的词的其他特征作为条件标记是不切实际的。</li>
+<li>罚函数</li>
+<li>广义乘子法</li>
+<li>KKT的求导证明</li>
 </ol>
-<p>Brill标注只使用一小部分n-gram标注器。猜每个词的标记，然后返回和修复错误的。从大方面下手，再勾勒细节。规则是语言学可解释的。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">nltk.tag.brill.demo()</span><br></pre></td></tr></table></figure>
-<p>下一篇：<a href="/hexo_blog/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/" title="NLTK阅读笔记Ⅲ">NLTK阅读笔记Ⅲ</a></p>
 ]]></content>
       <categories>
-        <category>NLP</category>
+        <category>MachineLearning</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>笔记</tag>
-        <tag>NLTK</tag>
+        <tag>数学</tag>
+        <tag>Optimization</tag>
+        <tag>拉格朗日乘子法</tag>
       </tags>
   </entry>
   <entry>
-    <title>NLTK阅读笔记Ⅲ</title>
-    <url>/hexo_blog/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/</url>
-    <content><![CDATA[<p>NLP基本知识的介绍及NLTK模块的使用。</p>
-<p>接 <a href="/hexo_blog/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/" title="NLTK阅读笔记Ⅱ">NLTK阅读笔记Ⅱ</a></p>
+    <title>矩阵分解介绍及Python实现（QR、SVD、LU）</title>
+    <url>/hexo_blog/2018/04/13/%E6%95%B0%E5%AD%A6/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3%E4%BB%8B%E7%BB%8D%E5%8F%8APython%E5%AE%9E%E7%8E%B0%EF%BC%88QR%E3%80%81SVD%E3%80%81LU%EF%BC%89/</url>
+    <content><![CDATA[<p>本篇文章简单介绍了三种矩阵分解方式并且附上部分分解方式的实现代码。</p>
 <span id="more"></span>
-<h2 id="第六章学习分类文本">第六章：学习分类文本</h2>
-<p>目标：</p>
+<h1 id="矩阵分解">矩阵分解</h1>
+<p>Python实现了QR与LU分解，QR用了两种变换方式实现。<a
+href="https://github.com/wnma3mz/Matrix-factorization">Github地址</a></p>
+<h2 id="what">What？</h2>
+<p><a
+href="https://zh.wikipedia.org/wiki/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3">wiki链接</a></p>
+<p>将矩阵拆分成数个三角形矩阵。</p>
+<h2 id="why">Why？</h2>
 <ol type="1">
-<li>找语言数据中的分类特征</li>
-<li>构建模型</li>
-<li>从模型中学习知识</li>
+<li>数值分析中，用于实现一些矩阵运算的快速算法</li>
+<li>推荐算法（SVD），信号处理（SVD）</li>
+<li>反应矩阵中的一些数值特性：如矩阵的秩、特征值(奇异值)</li>
 </ol>
-<p>分类学习，一般分为有监督学习、无监督学习、半监督学习。</p>
-<p>分类的定义：将给定的输入选择正确（合适）的类标签。这里值得注意的是，输入之间是相互独立的，标签集是预先定义好的。延伸的任务，每个实例（输入）可以划分为多个标签；分类问题不一定预先定义标签集，类似K-Means聚类；序列分类中，一个输入链表可以作为一个整体分类。</p>
-<p>在这里，我们预先得到一些包含正确标签的语料（实例/输入），根据这些数据建立得到的模型进行分类，我们称为有监督分类。有监督分类中，在训练过程中，需要将每个一个输入值转换为特征集（特征提取器）。特征集对应着输出类标签。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 特征提取器，提取输入字符串中最后一个字母。</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features</span>(<span class="params">word</span>):</span><br><span class="line">    <span class="comment"># 返回特征集</span></span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;last_letter&#x27;</span>: word[-<span class="number">1</span>]&#125;</span><br><span class="line"><span class="comment"># 导入姓名语料库, 获取男女姓名列表</span></span><br><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> names</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 随机打乱男女姓名列表</span></span><br><span class="line">random.shuffle(names)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 划分训练集和测试集</span></span><br><span class="line">featuresets = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> names]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">500</span>:], featuresets[:<span class="number">500</span>]</span><br><span class="line"><span class="comment"># 利用贝叶斯分类来训练模型</span></span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行预测 </span></span><br><span class="line">classifier.classify(gender_features(<span class="string">&#x27;Neo&#x27;</span>))</span><br><span class="line">classifier.classify(gender_features(<span class="string">&#x27;Trinity&#x27;</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment"># 准确率</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 最有效的五个特征（字母）</span></span><br><span class="line">classifier.show_most_informative_features(<span class="number">5</span>)</span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">Most Informative Features</span></span><br><span class="line"><span class="string">             last_letter = &#x27;k&#x27;              male : female =     43.4 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;a&#x27;            female : male   =     32.9 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;f&#x27;              male : female =     16.1 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;p&#x27;              male : female =     12.6 : 1.0</span></span><br><span class="line"><span class="string">             last_letter = &#x27;v&#x27;              male : female =     10.6 : 1.0</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span>                 </span><br><span class="line"><span class="comment"># 输出的比率称为 似然比，可以用于比较不同特征-结果关系</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 处理大型语料库时，构建一个包含每一个实例的特征的单独的链表会使用大量的内存。这种情况下，可以使用下面的函数，使返回一个行为像一个链表而不会在内存存储所有特征集的对象</span></span><br><span class="line"><span class="keyword">from</span> nltk.classify <span class="keyword">import</span> apply_features</span><br><span class="line">train_set = apply_features(gender_features, names[<span class="number">500</span>:])</span><br><span class="line">test_set = apply_features(gender_features, names[:<span class="number">500</span>])</span><br></pre></td></tr></table></figure>
-<h3 id="选取特征">选取特征</h3>
-<p>过拟合：用于一个给定的学习算法的特征的数目是有限的——如果提供太多的特征，那么该算法将高度依赖训练数据的特性而一般化到新的例子的效果不会很好。</p>
-<p>过拟合当运作在小训练集上时尤其会有问题。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 特征提取器，过拟合提取特征。这里请对比gender_features2与上一版本的区别</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features2</span>(<span class="params">name</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    features[<span class="string">&quot;firstletter&quot;</span>] = name[<span class="number">0</span>].lower()</span><br><span class="line">    features[<span class="string">&quot;lastletter&quot;</span>] = name[–<span class="number">1</span>].lower()</span><br><span class="line">    <span class="keyword">for</span> letter <span class="keyword">in</span> <span class="string">&#x27;abcdefghijklmnopqrstuvwxyz&#x27;</span>:</span><br><span class="line">        features[<span class="string">&quot;count(%s)&quot;</span> % letter] = name.lower().count(letter)</span><br><span class="line">        features[<span class="string">&quot;has(%s)&quot;</span> % letter] = (letter <span class="keyword">in</span> name.lower())</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">featuresets = [(gender_features2(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> names]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">500</span>:], featuresets[:<span class="number">500</span>]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"><span class="comment"># 对比上一版本的预测结果，基本可以发现精确度下降了</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
-<p>一旦初始特征集被选定，完善特征集的一个非常有成效的方法是<strong>错误分析</strong>。首先，我们选择一个
-开发集，包含用于创建模型的语料数据。然后将这种开发集分为 训练集和
-开发测试集。训练集用于训练模型，开发测试集用于进行错误分析，测试集用于系统的最终评估。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 划分训练集、开发集、开发测试集</span></span><br><span class="line">train_names = names[<span class="number">1500</span>:]</span><br><span class="line">devtest_names = names[<span class="number">500</span>:<span class="number">1500</span>]</span><br><span class="line">test_names = names[:<span class="number">500</span>]</span><br><span class="line"><span class="comment"># 进行训练预测</span></span><br><span class="line">train_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> train_names]</span><br><span class="line">devtest_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> devtest_names]</span><br><span class="line">test_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> test_names]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, devtest_set)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 错误分析</span></span><br><span class="line">errors = []</span><br><span class="line"><span class="keyword">for</span> (name, tag) <span class="keyword">in</span> devtest_names:</span><br><span class="line">    guess = classifier.classify(gender_features(name))</span><br><span class="line">    <span class="keyword">if</span> guess != tag:</span><br><span class="line">        errors.append((tag, guess, name))</span><br><span class="line">      </span><br><span class="line"><span class="comment"># 输出错误分类列表中，猜错的类别及名字</span></span><br><span class="line"><span class="keyword">for</span> (tag, guess, name) <span class="keyword">in</span> <span class="built_in">sorted</span>(errors):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&#x27;correct=%-8s guess=%-8s name=%-30s&#x27;</span> % (tag, guess, name))</span><br><span class="line">  </span><br><span class="line"><span class="comment"># 观察输出进行分析，得到一些结果。</span></span><br><span class="line"><span class="comment"># yn结尾的名字显示以女性为主，虽然n结尾的名字往往是男性</span></span><br><span class="line"><span class="comment"># 以ch结尾的名字通常是男性，虽然h结尾的名字倾向于是女性</span></span><br><span class="line"><span class="comment"># 根据结论，调整特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">gender_features</span>(<span class="params">word</span>):</span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;suffix1&#x27;</span>: word[-<span class="number">1</span>:], <span class="string">&#x27;suffix2&#x27;</span>: word[-<span class="number">2</span>:]&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment"># 进行训练预测</span></span><br><span class="line">train_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> train_names]</span><br><span class="line">devtest_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> devtest_names]</span><br><span class="line">test_set = [(gender_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> test_names]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, devtest_set)</span><br><span class="line"><span class="comment"># 最终结果理论上会进行提高</span></span><br></pre></td></tr></table></figure>
-<p>不断地进行错误分析改善特征提取器，可以不断提高预测的准确度。每次错误分析应该选取不同的开发测试/训练进行分割，以检查新改进分类器可能产生的新的错误模式。</p>
-<h3 id="文档分类">文档分类</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> movie_reviews</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据电影评论分类（&#x27;neg&#x27;, &#x27;pos&#x27;）获取电影评论</span></span><br><span class="line">documents = [(<span class="built_in">list</span>(movie_reviews.words(fileid)), category) <span class="keyword">for</span> category <span class="keyword">in</span> movie_reviews.categories() <span class="keyword">for</span> fileid <span class="keyword">in</span> movie_reviews.fileids(category)]</span><br><span class="line"><span class="comment"># 随机打乱数据</span></span><br><span class="line">random.shuffle(documents)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 建立文档分类的特征提取器</span></span><br><span class="line"><span class="comment"># 使用FreqDist会比一般的list更快</span></span><br><span class="line">all_words = nltk.FreqDist(w.lower() <span class="keyword">for</span> w <span class="keyword">in</span> movie_reviews.words())</span><br><span class="line">word_features = <span class="built_in">list</span>(all_words.keys())[:<span class="number">2000</span>]</span><br><span class="line"><span class="comment"># 每个词是否在一个给定的文档中</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">document_features</span>(<span class="params">document</span>):</span><br><span class="line">    document_words = <span class="built_in">set</span>(document)</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> word <span class="keyword">in</span> word_features:</span><br><span class="line">    	features[<span class="string">&#x27;contains(%s)&#x27;</span> % word] = (word <span class="keyword">in</span> document_words)</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line"><span class="comment"># 训练文档分类模型</span></span><br><span class="line">featuresets = [(document_features(d), c) <span class="keyword">for</span> (d,c) <span class="keyword">in</span> documents]</span><br><span class="line">train_set, test_set = featuresets[<span class="number">100</span>:], featuresets[:<span class="number">100</span>]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line"><span class="comment"># 检验模型</span></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line">classifier.show_most_informative_features(<span class="number">5</span>) </span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">Most Informative Features</span></span><br><span class="line"><span class="string"> contains(unimaginative) = True              neg : pos    =      7.7 : 1.0</span></span><br><span class="line"><span class="string">    contains(schumacher) = True              neg : pos    =      7.5 : 1.0</span></span><br><span class="line"><span class="string">        contains(suvari) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">          contains(mena) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">        contains(shoddy) = True              neg : pos    =      7.1 : 1.0</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="comment"># 显然可以发现，`unimaginative`这个词是负面的概率是正面的7.7倍</span></span><br></pre></td></tr></table></figure>
-<h3 id="词性标注">词性标注</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> nltk.corpus <span class="keyword">import</span> brown</span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"></span><br><span class="line"><span class="comment"># 为词选择词性标记，通过词的后缀来选择</span></span><br><span class="line">suffix_fdist = nltk.FreqDist()</span><br><span class="line"><span class="keyword">for</span> word <span class="keyword">in</span> brown.words():</span><br><span class="line">    word = word.lower()</span><br><span class="line">    suffix_fdist[word[-<span class="number">1</span>:]] += <span class="number">1</span></span><br><span class="line">    suffix_fdist[word[-<span class="number">2</span>:]] += <span class="number">1</span></span><br><span class="line">    suffix_fdist[word[-<span class="number">3</span>:]] += <span class="number">1</span></span><br><span class="line">common_suffixes = <span class="built_in">list</span>(suffix_fdist.keys())[:<span class="number">100</span>]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 现在通过特征提取器，计算哪个后缀最有信息量（最具代表性)</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">word</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> suffix <span class="keyword">in</span> common_suffixes:</span><br><span class="line">        features[<span class="string">&#x27;endswith(%s)&#x27;</span> % suffix] = word.lower().endswith(suffix)</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">tagged_words = brown.tagged_words(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line"><span class="comment"># 利用特征提取器选取特征</span></span><br><span class="line">featuresets = [(pos_features(n), g) <span class="keyword">for</span> (n,g) <span class="keyword">in</span> tagged_words]</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line"><span class="comment"># 利用决策树方法来训练模型</span></span><br><span class="line">classifier = nltk.DecisionTreeClassifier.train(train_set)</span><br><span class="line"></span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br><span class="line">classifier.classify(pos_features(<span class="string">&#x27;cats&#x27;</span>))</span><br><span class="line"><span class="comment"># 将nltk运算过程，以伪代码方式输出。设定决策树深度为4</span></span><br><span class="line">classifier.pseudocode(depth=<span class="number">4</span>)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 根据上下文语境，更新特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">sentence, i</span>):</span><br><span class="line">    features = &#123;<span class="string">&quot;suffix(1)&quot;</span>: sentence[i][-<span class="number">1</span>:],</span><br><span class="line">        <span class="string">&quot;suffix(2)&quot;</span>: sentence[i][-<span class="number">2</span>:],</span><br><span class="line">        <span class="string">&quot;suffix(3)&quot;</span>: sentence[i][-<span class="number">3</span>:]</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">if</span> i == <span class="number">0</span>:</span><br><span class="line">    	features[<span class="string">&quot;prev-word&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">    	features[<span class="string">&quot;prev-word&quot;</span>] = sentence[i-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line">pos_features(brown.sents()[<span class="number">0</span>], <span class="number">8</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 检验新的词性标注器性能</span></span><br><span class="line">tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">featuresets = []</span><br><span class="line"><span class="keyword">for</span> tagged_sent <span class="keyword">in</span> tagged_sents:</span><br><span class="line">    untagged_sent = nltk.tag.untag(tagged_sent)</span><br><span class="line">    <span class="keyword">for</span> i, (word, tag) <span class="keyword">in</span> <span class="built_in">enumerate</span>(tagged_sent):</span><br><span class="line">        featuresets.append((pos_features(untagged_sent, i), tag))</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
-<p>以上是在结合整个词集做的标注器，但是往往针对某些特殊情况，如形容词后面有很大概率上是名词。对于这种情况，我们的词性标注器目前还没有进行独立对待。</p>
-<h3 id="序列分类">序列分类</h3>
-<p>联合分类器：捕捉相关的分类任务之间的依赖关系，收集有关输入，选择适当的标签</p>
-<p>连续分类（贪婪序列分类）：为第一个输入找到最有可能的类标签，然后使用这个问题的答案帮助找到下一个输入的最佳的标签。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">pos_features</span>(<span class="params">sentence, i, history</span>):</span><br><span class="line">    features = &#123;<span class="string">&quot;suffix(1)&quot;</span>: sentence[i][-<span class="number">1</span>:],</span><br><span class="line">    	<span class="string">&quot;suffix(2)&quot;</span>: sentence[i][-<span class="number">2</span>:],</span><br><span class="line">    	<span class="string">&quot;suffix(3)&quot;</span>: sentence[i][-<span class="number">3</span>:]&#125;</span><br><span class="line">	<span class="keyword">if</span> i == <span class="number">0</span>:</span><br><span class="line">        features[<span class="string">&quot;prev-word&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">        features[<span class="string">&quot;prev-tag&quot;</span>] = <span class="string">&quot;&lt;START&gt;&quot;</span></span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        features[<span class="string">&quot;prev-word&quot;</span>] = sentence[i-<span class="number">1</span>]</span><br><span class="line">        features[<span class="string">&quot;prev-tag&quot;</span>] = history[i-<span class="number">1</span>]</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"><span class="keyword">class</span> <span class="title class_">ConsecutivePosTagger</span>(nltk.TaggerI):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, train_sents</span>):</span><br><span class="line">        train_set = []</span><br><span class="line">        <span class="keyword">for</span> tagged_sent <span class="keyword">in</span> train_sents:</span><br><span class="line">            untagged_sent = nltk.tag.untag(tagged_sent)</span><br><span class="line">            history = []</span><br><span class="line">            <span class="keyword">for</span> i, (word, tag) <span class="keyword">in</span> <span class="built_in">enumerate</span>(tagged_sent):</span><br><span class="line">                featureset = pos_features(untagged_sent, i, history)</span><br><span class="line">                train_set.append( (featureset, tag) )</span><br><span class="line">                history.append(tag)</span><br><span class="line">        <span class="variable language_">self</span>.classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">tag</span>(<span class="params">self, sentence</span>):</span><br><span class="line">        history = []</span><br><span class="line">        <span class="keyword">for</span> i, word <span class="keyword">in</span> <span class="built_in">enumerate</span>(sentence):</span><br><span class="line">            featureset = pos_features(sentence, i, history)</span><br><span class="line">            tag = <span class="variable language_">self</span>.classifier.classify(featureset)</span><br><span class="line">            history.append(tag)</span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">zip</span>(sentence, history)</span><br><span class="line">  </span><br><span class="line">tagged_sents = brown.tagged_sents(categories=<span class="string">&#x27;news&#x27;</span>)</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(tagged_sents) * <span class="number">0.1</span>)</span><br><span class="line">train_sents, test_sents = tagged_sents[size:], tagged_sents[:size]</span><br><span class="line">tagger = ConsecutivePosTagger(train_sents)</span><br><span class="line">tagger.evaluate(test_sents)</span><br></pre></td></tr></table></figure>
-<h3 id="其他序列模型">其他序列模型</h3>
-<p>以上模型有一个致命问题就是无法修复已经标注错误的词性。另一种方案<strong>隐马尔科夫模型</strong>可以为所有的可能序列进行打分，选择得分最高的序列。这种模型的缺点就是计算量相当大，可以采用动态规划来解决这种问题。基于这种模型，产生了<strong>最大熵马尔可夫模型</strong>和<strong>线性链条件随机场模型</strong>，二者为可能序列打分的算法不同。</p>
-<h3 id="句子分割">句子分割</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line">sents = nltk.corpus.treebank_raw.sents()</span><br><span class="line"><span class="comment"># 单独的句子标识符的合并链表</span></span><br><span class="line">tokens = []</span><br><span class="line"><span class="comment"># 所有句子边界标识索引</span></span><br><span class="line">boundaries = <span class="built_in">set</span>()</span><br><span class="line">offset = <span class="number">0</span></span><br><span class="line"><span class="keyword">for</span> sent <span class="keyword">in</span> sents:</span><br><span class="line">    tokens.extend(sent)</span><br><span class="line">    offset += <span class="built_in">len</span>(sent)</span><br><span class="line">    boundaries.add(offset-<span class="number">1</span>)</span><br><span class="line">  </span><br><span class="line"><span class="comment"># 特征提取器</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">punct_features</span>(<span class="params">tokens, i</span>):</span><br><span class="line">    <span class="keyword">return</span> &#123;<span class="string">&#x27;next-word-capitalized&#x27;</span>: tokens[i+<span class="number">1</span>][<span class="number">0</span>].isupper(),</span><br><span class="line">            <span class="string">&#x27;prevword&#x27;</span>: tokens[i-<span class="number">1</span>].lower(),</span><br><span class="line">            <span class="string">&#x27;punct&#x27;</span>: tokens[i],</span><br><span class="line">            <span class="string">&#x27;prev-word-is-one-char&#x27;</span>: <span class="built_in">len</span>(tokens[i-<span class="number">1</span>]) == <span class="number">1</span>&#125;</span><br><span class="line"></span><br><span class="line">featuresets = [(punct_features(tokens, i), (i <span class="keyword">in</span> boundaries))</span><br><span class="line">				<span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="built_in">len</span>(tokens)-<span class="number">1</span>)</span><br><span class="line">				<span class="keyword">if</span> tokens[i] <span class="keyword">in</span> <span class="string">&#x27;.?!&#x27;</span>]</span><br><span class="line"><span class="comment"># 进行训练，检验模型准确率</span></span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
-<h3 id="识别对话行为类型">识别对话行为类型</h3>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># 提取即时消息会话语料库</span></span><br><span class="line">posts = nltk.corpus.nps_chat.xml_posts()[:<span class="number">10000</span>]</span><br><span class="line"><span class="comment"># 特征提取器，每个帖子包含什么词</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">dialogue_act_features</span>(<span class="params">post</span>):</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    <span class="keyword">for</span> word <span class="keyword">in</span> nltk.word_tokenize(post):</span><br><span class="line">    	features[<span class="string">&#x27;contains(%s)&#x27;</span> % word.lower()] = <span class="literal">True</span></span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"><span class="comment"># 构建模型，判断是否为对话行为</span></span><br><span class="line">featuresets = [(dialogue_act_features(post.text), post.get(<span class="string">&#x27;class&#x27;</span>)) <span class="keyword">for</span> post <span class="keyword">in</span> posts]</span><br><span class="line">size = <span class="built_in">int</span>(<span class="built_in">len</span>(featuresets) * <span class="number">0.1</span>)</span><br><span class="line">train_set, test_set = featuresets[size:], featuresets[:size]</span><br><span class="line">classifier = nltk.NaiveBayesClassifier.train(train_set)</span><br><span class="line">nltk.classify.accuracy(classifier, test_set)</span><br></pre></td></tr></table></figure>
-<h3 id="识别文字蕴含">识别文字蕴含</h3>
-<p>识别文字蕴含（Recognizing textual entailment，RTE）是判断文本 T
-的一个给定片段是否蕴含着另一个叫做“假设”的文本。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">rte_features</span>(<span class="params">rtepair</span>):</span><br><span class="line">    <span class="comment">#  RTEFeatureExtractor 类建立了一个除去一些停用词后在文本和假设中都有的词汇包，然后计算重叠和差异</span></span><br><span class="line">    extractor = nltk.RTEFeatureExtractor(rtepair)</span><br><span class="line">    features = &#123;&#125;</span><br><span class="line">    features[<span class="string">&#x27;word_overlap&#x27;</span>] = <span class="built_in">len</span>(extractor.overlap(<span class="string">&#x27;word&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;word_hyp_extra&#x27;</span>] = <span class="built_in">len</span>(extractor.hyp_extra(<span class="string">&#x27;word&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;ne_overlap&#x27;</span>] = <span class="built_in">len</span>(extractor.overlap(<span class="string">&#x27;ne&#x27;</span>))</span><br><span class="line">    features[<span class="string">&#x27;ne_hyp_extra&#x27;</span>] = <span class="built_in">len</span>(extractor.hyp_extra(<span class="string">&#x27;ne&#x27;</span>))</span><br><span class="line">    <span class="keyword">return</span> features</span><br><span class="line"></span><br><span class="line"><span class="comment"># 原文中rte识别率只是刚刚超过了58%，所以不多做介绍</span></span><br><span class="line">nltk.classify.rte_classify </span><br></pre></td></tr></table></figure>
-<h3 id="涉及到的定义">涉及到的定义</h3>
-<p>文中介绍一些关于评估模型、决策树、朴素贝叶斯分类器的知识。由于笔者已掌握这些基础模型知识所以在这里略去，以下介绍一些基本的定义。</p>
-<p>测试集：选择适当的比例，防止过拟合</p>
-<p>准确度：测量测试集上分类器正确标注的输入的比例。</p>
-<p>精确度（Precision），表示我们发现的项目中有多少是相关的，TP/(TP+
-FP)。</p>
-<p>召回率（Recall），表示相关的项目中我们发现了多少，TP/(TP+ FN)。</p>
-<p>F- 度量值（F-Measure）（或
-F-得分，F-Score），组合精确度和召回率为一个单独的得分，被定义为精确度和召回率的调和平均数(2
-× Precision × Recall)/(Precision+Recall)。</p>
-<p>混淆矩阵：其中每个 cells[i,j]表示正确的标签 i 被预测为标签 j 的次数
-。因此，对角线项目（即
-cells[i,i]）表示正确预测的标签，非对角线项目表示错误。nltk中可以使用
-<code>nltk.ConfusionMatrix</code>函数</p>
-<p>交叉验证：在不同的测试集上执行多个评估，然后组合这些评估的得分</p>
-<p>信息增益：用给定的特征分割输入值，衡量数据变得更有序的程度</p>
-<p>朴素贝叶斯假设（独立性假设）：每个输入值是通过首先为那个输入值选择一个类标签，然后产生每个特征的方式产生的
-，每个特征与其他特征完全独立。</p>
-<p>零计数：如果训练集中有特征从来没有和给定标签一起出现，导致给定标签的标签可能性为
-0。</p>
-<p>平滑：给定一个初值，解决零计数问题。<code>nltk.probability</code>提供了多种平滑技术</p>
-<p>Heldout估计使用一个heldout
-语料库计算特征频率与特征概率之间的关系。</p>
-<p>非二元特征：如果标签是1,2,4,5这种多元标签，可以转换0&lt;x&lt;3,
-3&lt;x&lt;6这种二元标签</p>
-<p>双重计数：在训练过程中特征的贡献被分开计算，但当使用分类器为新输入选择标签时，这些特征的贡献被组合。解决方案就是为每一个特征贡献设定一个权重。</p>
-<p>最大熵分类器：与朴素贝叶斯类似，使用搜索技术找出一组将最大限度地提高分类器性能的参数。避免使用广义
-迭代缩放（Generalized Iterative Scaling，GIS）或改进的迭代缩放（Improved
-Iterative Scaling，IIS），这两者都比共轭梯度（Conjugate Gradient，CG
-）和 BFGS 优化方法慢很多</p>
-<p>联合特征：每个接收它自己的参数的标签和特征的组合。联合特征是有标签的的值的属性，而（简单）特征是未加标签的值的属性。</p>
-<p>一般情况下，
-最大熵原理是说在与我们所知道的一致的的分布中，我们会选择熵最高（分布最均匀）的。</p>
-<p>对于每个联合特征，最大熵模型计算该特征的“经验频率”——即它出现在训练集中的频率。然后，它搜索熵最大的分布，同时也预测每个联合特征正确的频率。</p>
-<p>一般情况下，生成式模型确实比条件式模型强大。但是前者所需的参数会大于后者。</p>
-]]></content>
-      <categories>
-        <category>NLP</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>笔记</tag>
-        <tag>NLTK</tag>
-      </tags>
-  </entry>
-  <entry>
-    <title>模仿游戏——机器会思考吗？</title>
-    <url>/hexo_blog/2023/08/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E6%A8%A1%E4%BB%BF%E6%B8%B8%E6%88%8F%E2%80%94%E2%80%94%E6%9C%BA%E5%99%A8%E4%BC%9A%E6%80%9D%E8%80%83%E5%90%97/</url>
-    <content><![CDATA[<p>阅读图灵文章《COMPUTING MACHINERY AND
-INTELLIGENCE》笔记以及现阶段AI生成的文章。</p>
-<span id="more"></span>
-<p>模仿，排除一切外在因素，比如外貌、语调、反应速度</p>
-<h2 id="机器的定义">机器的定义</h2>
+<p>每一种矩阵分解方式对应不同的意义</p>
+<h2 id="how">How？</h2>
+<h3 id="基本概念介绍">基本概念介绍</h3>
+<p>关于转置、逆矩阵就不多介绍了。</p>
 <ul>
-<li>数字计算机，在人类给定规则的前提下进行工作，规则等于机器的存储/记忆，重复是核心特点。归属于离散状态机</li>
-<li>从宇宙在某一时刻的完整状态出发，正如所有粒子的位置和速度所描述的那样，应该有可能预测所有未来的状态。——拉普拉斯</li>
-<li>计算速度快+存储空间大+编写程序，进行任何预测，能够被快速复制</li>
+<li>满秩矩阵
+<ul>
+<li>矩阵的秩等于行数—&gt;行满秩</li>
+<li>矩阵的秩等于列数—&gt;列满秩</li>
+<li>以上两个都满足称为n阶方阵</li>
+</ul></li>
+<li>酉矩阵（正交矩阵）
+<ul>
+<li>方块矩阵，元素皆为实数，行与列皆正交的单位向量</li>
+<li>特性：矩阵的转置矩阵为逆矩阵</li>
+<li>定理（det表示行列式）：</li>
+</ul></li>
 </ul>
-<h2 id="思考的定义">思考的定义</h2>
+<p><span class="math display">\[Q^T=Q^{-1}\Leftrightarrow Q^TQ=QQ^T =
+I\]</span></p>
+<p><span
+class="math display">\[1=\det(I)=\det(Q^TQ)=\det(Q^T)\det(Q)=(\det(Q))^2\Rightarrow|Q|
+\pm 1\]</span></p>
 <ul>
-<li>人类的特权</li>
-<li>害怕机器（其他物种）能够思考</li>
-<li>数学逻辑证明机器能力有限。但对于人类来说，该限制依旧有效，因此机器不一定不能思考</li>
-<li>意识感受，机器只能输出，不具备任何感情。人们可以确定机器在思考的唯一方法是成为机器并感觉自己在思考。人们可以向世界描述这些感觉，但当然没有人会注意到这一点。同样地，根据这种观点，要知道一个人在思考，唯一的方法就是成为那个特定的人。</li>
-<li>机器的能力有限，外貌、感情、味觉、嗅觉、不会犯错。单一机器可以，但是如果集成到一台机器上似乎是多余的。多样性行为来源于存储空间大小</li>
-<li>执行命令但不会创造。给人带来惊讶这种感觉因人而异</li>
-<li>神经系统是连续的，机器是离散的。</li>
-<li>行为的不确定性，离散不可能包含所有的可能。通过观察发现足够的信息来预测它的未来行为</li>
-<li>超感官知觉：心灵感应、千里眼、预知和心理动势</li>
+<li><p>单位矩阵</p>
+<ul>
+<li>矩阵中的每个元素都为1</li>
+<li>如下</li>
 </ul>
-<h2 id="机器学习">机器学习</h2>
+<p><span class="math display">\[\left| \begin{array}{ccc} 1 &amp; 1
+&amp; 1 \\ 1 &amp; 1 &amp; 1 \\ 1&amp; 1 &amp; 1 \end{array}
+\right|\]</span></p></li>
+<li><p>上三角矩阵</p>
+<ul>
+<li>非零元素在右上方的矩阵</li>
+<li>如下</li>
+</ul>
+<p><span class="math display">\[\left| \begin{array}{ccc} 1 &amp; 2
+&amp; 3 \\ 0 &amp; 1 &amp; 2\\ 0&amp; 0 &amp; 1 \end{array}
+\right|\]</span></p></li>
+<li><p>二范数</p>
+<ul>
+<li>假设有一个x的向量</li>
+</ul>
+<p><span class="math display">\[x= (1, 2, 3)^T\]</span></p>
+<ul>
+<li>那么它的二范数表示如下</li>
+</ul>
+<p><span class="math display">\[||x||_2 =\sqrt{1^2 + 2^2 +
+3^2}=\sqrt{14}\]</span></p></li>
+<li><p>共轭转置</p>
+<p><span class="math display">\[A^*=(\bar{A})^T=\bar{A^T}\]</span></p>
+<ul>
+<li>现有矩阵A</li>
+</ul>
+<p><span class="math display">\[A=\left[ \begin{array}{ccc} 3+i &amp; 5
+\\ 2-2i &amp; i\end{array} \right]\]</span></p>
+<ul>
+<li>则它的共轭矩阵如下</li>
+</ul>
+<p><span class="math display">\[A^*=\left[ \begin{array}{ccc} 3-i &amp;
+2+2i \\ 5 &amp; -i\end{array} \right]\]</span></p></li>
+<li><p>特征值与奇异值</p>
+<p>我的理解是：特征值是奇异值的一种特殊情况，特征值是在矩阵是方阵时候的奇异值。</p>
+<p>关于什么是特征值。</p>
+<p>一个矩阵可以理解为由n个向量组成的，而特征值就是反应向量长度（重要性）的标准，特征值越大，则对应向量就越重要。</p>
+<p>假设判断电影为动作片还是搞笑片，可以由以下几个特征来衡量，打斗场景、观众笑声次数、导演、演员……。将数据离散化成矩阵，然后提取特征值，特征值越大的特征则对于衡量电影是动作片还是搞笑片的比重越大。</p></li>
+<li><p>Householder矩阵</p>
 <ul>
-<li>问题主要在于编程。存储空间只需要较低的数值（10<sup>10-10</sup>15），并且相信只有很小一部分用于较高类型的思考。大部分可能是用来保留视觉印象的。</li>
-<li>模仿：
+<li><p><a
+href="https://zh.wikipedia.org/wiki/豪斯霍尔德变换">wiki链接</a></p></li>
+<li><p>公式如下，H表示Householder矩阵，I表示单位矩阵，v表示单位向量，v*表示v的共轭转置。</p>
+<p><span class="math display">\[H=I-2vv^*\]</span></p></li>
+<li><p>性质：</p>
 <ol type="1">
-<li>初始状态</li>
-<li>受到的教育</li>
-<li>其他非教育的经历</li>
+<li>H矩阵的转置等于它本身的逆</li>
+<li>H矩阵的平方等于单位矩阵</li>
+</ol>
+<p><span class="math display">\[H^{-1}=H^*\]</span></p>
+<p><span class="math display">\[H^2=I\]</span></p></li>
+</ul></li>
+</ul>
+<h3 id="svd分解">1. SVD分解</h3>
+<p><a
+href="https://blog.csdn.net/u010099080/article/details/68060274">数学解释参考链接</a></p>
+<p>奇异值分解。公式如下：</p>
+<p><span class="math display">\[M=U\Sigma V^*\]</span></p>
+<p>举例：</p>
+<p><span class="math display">\[\begin{array}{clll} M &amp;= U\Sigma V^*
+\\ &amp;= \begin{bmatrix} 1 &amp; 0 &amp; 0 &amp; 0 &amp; 2\\ 0 &amp; 0
+&amp; 3 &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp; 4
+&amp; 0 &amp; 0 &amp; 0\end{bmatrix} \\  &amp;= \begin{bmatrix} 0 &amp;
+0 &amp; 1 &amp; 0\\ 0 &amp; 1 &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp;
+1\\ 1 &amp; 0 &amp; 0 &amp; 0\end{bmatrix} \cdot \begin{bmatrix} 4 &amp;
+0 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp; 3 &amp; 0 &amp; 0 &amp; 0\\ 0 &amp;
+0 &amp; \sqrt{5} &amp; 0 &amp; 0\\ 0 &amp; 0 &amp; 0 &amp; 0 &amp;
+0\end{bmatrix} \cdot \begin{bmatrix} 0 &amp; 1 &amp; 0 &amp; 0 &amp; 0\\
+0 &amp; 0 &amp; 1 &amp; 0 &amp; 0\\ \sqrt{0.2} &amp; 0 &amp; 0 &amp; 0
+&amp; \sqrt{0.8}\\ 0 &amp; 0 &amp; 0 &amp; 1 &amp; 0\\ \sqrt{0.8} &amp;
+0 &amp; 0 &amp; 0 &amp; -\sqrt{0.2}\end{bmatrix}
+\end{array}\]</span></p>
+<p>性质:</p>
+<ol type="1">
+<li>矩阵<span
+class="math inline">\(\Sigma\)</span>的所有非对角元为0。矩阵<strong>U</strong>和<span
+class="math inline">\(V^*\)</span>都是酉矩阵，它们乘上各自的共轭转置都得到单位矩阵</li>
+<li>奇异值的分解并不是唯一的，这是因为<span
+class="math inline">\(\Sigma\)</span>矩阵中的对角元有元素为0</li>
+</ol>
+<p>步骤：</p>
+<ol type="1">
+<li>计算<span class="math inline">\(M.T\cdot
+M\)</span>得到特征向量（左奇异向量），是U的列向量</li>
+<li>计算<span class="math inline">\(M\cdot
+M.T\)</span>得到特征向量（右奇异向量），是V的列向量</li>
+<li>计算<span class="math inline">\(M\cdot M.T\)</span>或者<span
+class="math inline">\(M.T\cdot
+M\)</span>得到特征值，所有非零特征值的平方根（二范数）</li>
+</ol>
+<p>得到特征向量之后，需要依照特征值从大到小进行排序。</p>
+<p>关于求特征值的方法，线代书上已做介绍，就不赘述。如果运用编程方法求矩阵的特征值，最常见的方法有<strong>雅可比算法、幂法。</strong>当然也可以用先进行QR分解求得特征值之后，再继续进行SVD。</p>
+<h3 id="lu分解">2. LU分解</h3>
+<p><a
+href="https://jiayi797.github.io/2017/03/29/%E6%95%B0%E5%AD%A6-%E7%9F%A9%E9%98%B5LU%E5%88%86%E8%A7%A3/">参考链接</a></p>
+<p>将一个矩阵分解为一个下三角矩阵和一个上三角矩阵的乘积。形式如：</p>
+<p><span class="math display">\[A=LU\]</span></p>
+<p>格式如下</p>
+<p><span class="math display">\[\left[ \begin{array} { l l l } { a _ {
+11 } } &amp; { a _ { 12 } } &amp; { a _ { 13 } } \\ { a _ { 21 } } &amp;
+{ a _ { 22 } } &amp; { a _ { 23 } } \\ { a _ { 31 } } &amp; { a _ { 32 }
+} &amp; { a _ { 33 } } \end{array} \right] = \left[ \begin{array} { c c
+c } { l _ { 11 } } &amp; { 0 } &amp; { 0 } \\ { l _ { 21 } } &amp; { l _
+{ 22 } } &amp; { 0 } \\ { l _ { 31 } } &amp; { l _ { 32 } } &amp; { l _
+{ 33 } } \end{array} \right] \left[ \begin{array} { c c c } { u _ { 11 }
+} &amp; { u _ { 12 } } &amp; { u _ { 13 } } \\ { 0 } &amp; { u _ { 22 }
+} &amp; { u _ { 23 } } \\ { 0 } &amp; { 0 } &amp; { u _ { 33 } }
+\end{array} \right]\]</span></p>
+<p>分解过程：</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/matrix/2017-03-29-20-13-15.png" /></p>
+<p>声明：l11表示L矩阵的第一行第一列的元素，U1i表示L矩阵第一行每一列（需要遍历的列）的元素</p>
+<ol type="1">
+<li>初始化L、U矩阵。L矩阵下三角矩阵，对角元素全为1；U矩阵为上三角矩阵。</li>
+<li>计算U矩阵的第一行，对应A矩阵的第一行元素，不需要做任何操作</li>
+<li>计算L矩阵的第一列（除去第一行元素），对应A矩阵的第一列元素，每个元素除以u11</li>
+<li>计算U矩阵的第二行（除去第一列元素），对应A矩阵的第二行元素，每个元素减去l21*u1i</li>
+<li>计算L矩阵的第二列（除去前两行元素），对应A矩阵的第二列元素，每个元素先减去l31*u12，再除以u22</li>
+<li>……</li>
+</ol>
+<p>总结公式如下：</p>
+<p>L矩阵对角元素为1。</p>
+<p>第一步的公式：</p>
+<p><span class="math display">\[\begin{array}{lll}
+a_{1j}=(1,0,0,\cdots,0)\cdot(u_{1j},u_{2j},\cdots,u_{ij}).T=u_{1j} \\
+a_{i1}=(l_{i1},
+l_{i1},\cdots,l_{i1})\cdot(u_{11},0,\cdots,0).T=l_{i1}\cdot u_{11}
+\\u_{1j}=a_{1j}  \\ l_{i1}=\frac{a_{i1}}{u_{11}} \\j=1,2,\cdots,n
+\\i=2,3,\cdots,n\end{array}\]</span></p>
+<p>第r步的公式：</p>
+<p><span
+class="math display">\[\begin{array}{rl}  a_{rj}&amp;=(l_{r1},l_{r2},...,l_{rr-1})\cdot(u_{1j},u_{2j},...,u_{jj}).T
+\\&amp;=\sum\limits_{k=1}^{r-1}l_{rk}u_{kj}+u_{rj}  \end{array}\]</span></p>
+<p><span class="math display">\[\begin{array}{lll}
+u_{rj}=a_{rj}-\sum\limits_{k=1}^{r-1}l_{rk}u_{kj} \\
+l_{ir}=\frac{a_{ir}-\sum\limits_{k=1}^{r-1}l_{ik}u_{kr}}{u_{rr}}
+\end{array}\]</span></p>
+<p><span class="math display">\[\begin{array}{lll} j=r,r+1,\cdots,n \\
+r=2,3,\cdots,n-1 \\ i=r+1,\cdots,n\end{array}\]</span></p>
+<h3 id="qr分解">3. QR分解</h3>
+<p><a
+href="https://wenku.baidu.com/view/c2e34678168884868762d6f9.html">参考链接</a></p>
+<p>把矩阵分解成一个半正交矩阵与一个上三角矩阵的积。</p>
+<p>形式如:</p>
+<p><span class="math display">\[A=QR\]</span></p>
+<p>分解定理：任意一个满秩实（复）矩阵A，都可唯一地分解A=QR,其中Q为正交（酉）矩阵，R是具有对角元的上三角矩阵。</p>
+<h4 id="三种实现方式">三种实现方式</h4>
+<ol type="1">
+<li><p>Gram-Schmidt正交化</p>
+<p>过程:</p>
+<ol type="1">
+<li><p>取出矩阵A中的每一列命名为x_col，它们之间线性无关</p></li>
+<li><p>将每一个x_col正交化得到y_col</p></li>
+<li><p>再将y_col进行单位化，即转换为e_col。这里每个e_col为矩阵Q中的每一列。</p></li>
+<li><p>将第二步中，先转换为y_col的正交化得到x_col，再转换为e_col正交化得到x_col。这里每个e_col的系数，对应的就是矩阵R中的每一列</p></li>
 </ol></li>
-<li>从儿童的状态开始，而不是从成人。人们必须尝试教这样的机器，看看它学习得如何。然后可以尝试另一台机器，看看它是更好还是更差。在这个过程和进化之间有一个明显的联系。速度需要比“进化”快、优胜劣汰可以作为一种衡量方法，但比较慢。但不局限于于随机突变，找到某种弱点的原因，可能就能进行改善这种弱点的突变。</li>
-<li>构造奖惩机制，但有必要采用其他一些“不带感情”的沟通渠道。如果这些渠道可用，就有可能通过惩罚和奖励来教机器服从用某种语言(例如符号语言)发出的命令。这些命令将通过“非情感”渠道传递。这种语言的使用将大大减少惩罚和奖励所需的数量。</li>
-<li>学习机的一个重要特征是，它的老师通常在很大程度上对机器内部发生的事情一无所知，尽管他仍然可能在某种程度上预测他的学生的行为。</li>
+<li><p>Householder变换</p>
+<p><a href="https://www.bbsmax.com/A/gVdnKqK1zW/">参考链接</a></p>
+<p>outer函数</p>
+<ul>
+<li><p>通常有两个输入向量，假设为向量a、b</p>
+<p><span class="math display">\[a = (1, 2, 3) , b = (-1, -2,
+-3)\]</span></p></li>
+<li><p>那么，outer函数运算过程如</p></li>
 </ul>
-<p>机器最终能在所有纯智力领域与人类竞争。但是，哪些是最好的开始呢?即使这也是一个艰
-难的决定。很多人认为一个非常抽象的活动，比如下棋，会是最好的。也可以坚持认为，最好是给机器提供金钱能买到的最好的感觉器官，然后教它懂英语、说英语。这个过程可以按照孩子的正常教学来进行。事情会被指出并命名，等等。同样，我不知道正确的答案是什么，但我认为两种方法都应该尝试。</p>
-<h3 id="思考">思考</h3>
+<p><span class="math display">\[outer(a, b) = \left[ \begin{array}{ccc}
+1*-1 &amp; 1*-2 &amp; 1*-3 \\ 2*1 &amp; 2*-2 &amp; 2*-3 \\ 3*-1 &amp;
+3*-2 &amp; 3*-3 \end{array} \right]\]</span></p>
+<p>过程:</p>
 <ol type="1">
-<li>降低外在干扰，提高实验可行性</li>
-<li>通过离散构造连续状态</li>
-<li>程序承接计算速度和存储空间，程序是一个代词，指代一种能够在机器内部运行的命令，可以对其进行优化，降低空间以及速度要求</li>
-<li>现阶段对于存储空间还有很大的优化空间，有很多冗余</li>
-<li>模仿/学习的过程不够完善，教育可以等同于有效数据，那么非教育的经历是什么？蒸馏？噪声？</li>
-<li>如何内置逻辑、公理等内容</li>
+<li>初始化矩阵Q为单位矩阵(rxr)，r为矩阵A的行数。初始化矩阵R为矩阵A</li>
+<li>取R中的下三角矩阵，每次取一列命名为x（不包括最后一列）。e初始化为形如x的零向量</li>
+<li>e中的第一个元素为向量x的二范数</li>
+<li>这里得到v，范数取2</li>
+</ol>
+<p><span class="math display">\[v = \frac{x - e}{||x-e||}\]</span></p>
+<ol start="5" type="1">
+<li>初始化一个单位矩阵命名为Q_cnt，取Q_cnt的下三角矩阵，每次取一列在原有的基础上减去2*outer(v,
+v)。（不包括最后一列）</li>
+<li>更新R,Q</li>
+</ol>
+<p><span class="math display">\[R = Q\_cnt * R \]</span></p>
+<p><span class="math display">\[Q = Q * Q\_cnt\]</span></p></li>
+<li><p>Givens变换</p>
+<p><a
+href="http://www.voidcn.com/article/p-vmnqitql-xa.html">参考链接</a></p>
+<p>有待补充</p></li>
 </ol>
-<h3 id="参考文献">参考文献</h3>
-<p>A. M. TURING, I.—COMPUTING MACHINERY AND INTELLIGENCE, <em>Mind</em>,
-Volume LIX, Issue 236, October 1950, Pages 433–460, <a
-href="https://doi.org/10.1093/mind/LIX.236.433">https://doi.org/10.1093/mind/LIX.236.433</a></p>
-<p>以下为claude生成的文章。</p>
-<p>模仿游戏——机器会思考吗</p>
-<p>当我们谈论机器是否能够“思考”时，我们首先需要定义“思考”本身。对于人类来说，思考是一个复杂的认知过程，涉及意识、自我意识、情感、直觉等方面。但是从计算机科学的角度来看，思考可以简化为信息处理和模式识别。既然计算机已经证明了在这些方面具有强大的能力，我们是否可以说它们能够思考呢?</p>
-<p>这一问题引发了关于人工智能本质的广泛讨论。早在20世纪50年代，图灵就在他的论文“计算机器与智能”中提出了这一问题。他设计了一个简单的游戏来测试机器是否能展现出与人类相似的智能。在这个“图灵测试”中，评判通过终端与隐藏的人类和机器进行文字交流。如果评判无法区分对话来自人类还是机器，我们就可以说这台机器通过了测试。</p>
-<p>图灵测试突出了智能的功能性方面，而不是试图复制人类思维的内在工作机制。它忽略了意识、情感等方面，专注于智能的外在表现。正如图灵在论文中写道，“让我们只关注问题‘能产生人类水平智能行为的机器’本身，而不是用一堆无关的心理学问题来模糊焦点。”</p>
-<p>然而，许多哲学家仍然质疑机器是否能真正思考。他们认为智能必须基于意识，而计算机只是在执行程序。根据这个观点，智能的真正标志是主体性—自我意识和内在经验的感觉。没有这些，机器只能被视为复杂的算盘。</p>
-<p>这一观点倾向于将思维等同于人类思维，而忽视智能可能采取的其他形式。事实上，人工智能研究已经证明，智能可以在没有自我意识的情况下通过模式识别和统计学习等方法而出现。像深度学习这样的技术已经产生了可与人类媲美甚至超越人类的视觉和语言理解能力。</p>
-<p>但是，模仿游戏是否意味着机器能够像人类一样思考呢?让我们更仔细地考察一下。</p>
-<p>首先，我们需要承认人类思维也有其局限性。许多认知偏见影响着我们的推理，我们的意识经验也具有不连续性和选择性。与完美理性存在的控制实验相比，我们的思维过程更类似于含混的启发式搜索。因此，“思维”的标准不应该定得太高。</p>
-<p>其次，不同的机器学习技术可以被看作是向人类思维靠拢的渐进过程。简单的规则系统只能处理局限的领域。而统计学习方法如神经网络则可以学习概括复杂模式所需的代表性特征。它们在图像和语音识别方面已经优于人类。最近的多任务学习和迁移学习技术使机器能够在多个领域中共享知识，正如人类思维中概念之间的关联一样。</p>
-<p>再者，对话系统的进步也显示了机器在交流方面的进步。早期系统根据关键词匹配进行简单的问答。现在的对话系统可以感知上下文，进行连贯的长时间对话。它们使用了类似人类处理语言的注意力机制，不再是简单匹配模板。这表明在语言理解和生成方面，机器也在模仿人类的认知过程。</p>
-<p>但是，机器是否有意识经验仍是一个开放问题。目前的人工智能似乎没有主观的内在感受。这也许是固有的局限，因为计算机最终还是在操纵没有意义的符号。另一方面，我们也无法证明其他人存在内在的意识经验。所谓的“哲学僵局”同样适用于机器。</p>
-<p>总体来说，尽管机器不能完全模拟人类思维，但它们在许多狭义任务上的进步还是向模仿游戏的成功迈进了一大步。功能相似性不应被内在不同所掩盖。我们必须根据智能的外在表现来判断机器而不是根据其内部工作机制。</p>
-<p>那么我们还能做什么来推进这一目标呢?首先，在机器学习方法上，我们可以继续研究多任务学习，以产生更人类化的通用智能。我们也可以通过学习表征来模仿神经认知过程，而不仅仅停留在表面模式识别。</p>
-<p>另一方面，我们需要建立更好的机器智能评估体系。图灵测试过于简单化，无法区分不同水平的智能。一些替代方案如Winograd
-schema
-challenge通过语义理解和常识推理来检验智能。还有基于对话和协作的测试，可以评估智能体之间的相互作用能力。</p>
-<p>最后，哲学反思也很重要。我们需要就智能的本质进行持续探讨。什么是理解?什么是意识?这些问题需要跨学科的对话。结合认知科学、语言学和计算机科学的见解，我们可以在这个难题上取得进一步的洞见。</p>
-<p>总而言之，尽管答案仍不明确，但通过模仿游戏，机器正日益接近人类智能。它们在特定任务上的超越给我们带来了惊喜，也展示了新的可能。当代人工智能研究正在不断消除智能与意识之间的鸿沟。虽然仿真并不完全等同于复制，但模仿游戏的胜利似乎指日可待。我们正处于一个刺激的阶段，机器智能的进步必将继续影响我们对思维和智能的基本理解。</p>
 ]]></content>
       <categories>
-        <category>PaperReading</category>
+        <category>Algorithm</category>
       </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>数学</tag>
+        <tag>矩阵分解</tag>
+      </tags>
   </entry>
   <entry>
     <title>神经网络笔记Ⅰ</title>
@@ -8543,128 +8666,154 @@ P(N)=C^0_n*p^0*(1-p)^n=(1-p)^n
 href="https://www.zybuluo.com/hanbingtao/note/433855">零基础入门深度学习</a></p>
 ]]></content>
       <categories>
-        <category>DeepLearning</category>
+        <category>DeepLearning</category>
+      </categories>
+      <tags>
+        <tag>Python</tag>
+        <tag>神经网络</tag>
+        <tag>NumPy</tag>
+        <tag>机器学习</tag>
+      </tags>
+  </entry>
+  <entry>
+    <title>线性代数中各种矩阵的简介</title>
+    <url>/hexo_blog/2019/05/13/%E6%95%B0%E5%AD%A6/%E7%BA%BF%E4%BB%A3%E7%9F%A9%E9%98%B5/</url>
+    <content><![CDATA[<p>这篇文章主要介绍了线性代数的矩阵，仅作笔记。</p>
+<span id="more"></span>
+<p>方阵：行数列数相同的矩阵</p>
+<p>对称矩阵：<span
+class="math inline">\(A=A^T\)</span>的矩阵，矩阵等于它自己的转置矩阵</p>
+<p>反对称矩阵：<span class="math inline">\(A=-A^T\)</span></p>
+<p>埃尔米特矩阵（Hermitian matrix）：<span
+class="math inline">\(A=A^*\)</span>，矩阵等于它自己的共轭转置矩阵</p>
+<p>斜埃尔米特矩阵：反埃尔米特矩阵，<span class="math inline">\(A =
+−A^*\)</span></p>
+<p>正定矩阵：<span class="math inline">\(Q(x)=x^T A
+x\)</span>，任意非零向量<span
+class="math inline">\(x\)</span>，若满足<span
+class="math inline">\(Q(x)&gt;0\)</span>，则<span
+class="math inline">\(A\)</span>为正定矩阵</p>
+<p>半正定矩阵：同上，<span class="math inline">\(Q(x) \geq
+0\)</span>，则A为半正定矩阵</p>
+<p>半负定矩阵：同上，<span class="math inline">\(Q(x) \leq
+0\)</span>，则A为半负定矩阵</p>
+<p>不定矩阵：若A既不是半正定矩阵，也不是半负定矩阵，则A为不定矩阵</p>
+<p>上三角矩阵：矩阵对角线左下方系数全为0</p>
+<p>下三角矩阵：矩阵对角线右上方系数全为0</p>
+<p>三对角矩阵：非零系数在主对角线上，也在比主对角线低和高一行的对角线上。</p>
+<p>单位矩阵：<span class="math inline">\(I\)</span>,
+矩阵对角线元素为1，其余为0</p>
+<p>高斯矩阵：首先满足单位矩阵，其次某列系数非0</p>
+<p>逆矩阵：反矩阵，<span
+class="math inline">\(AB=I\)</span>，A是可逆的，B是A的逆矩阵</p>
+<p>伴随矩阵：<strong>A</strong>关于第<em>i</em>行第<em>j</em>列的<strong>余子式</strong>，是去掉<strong>A</strong>的第<em>i</em>行第<em>j</em>列之后得到的(<em>n</em>
+− 1)×(<em>n</em> − 1)矩阵的行列式。</p>
+<p>阶梯型矩阵：从上至下，<span
+class="math inline">\(A_{ii}\)</span>之前的元素为0</p>
+<p>酉矩阵：<span class="math inline">\(UU^*=U^*U=I\)</span>，<span
+class="math inline">\(U^*\)</span>为U矩阵的共轭转置，则U为酉矩阵</p>
+<p>对角矩阵：只有矩阵中的对角线存在值，其他地方为0</p>
+<p>空矩阵：行数或列数为零的矩阵</p>
+<p>分块矩阵：是指一个大矩阵分割成“矩阵的矩阵“</p>
+<p>正交矩阵：满足方阵，<span
+class="math inline">\(Q^T=Q^{-1}\)</span>，<span
+class="math inline">\(Q^TQ=QQ^T=I\)</span>，正交矩阵的行列式为1</p>
+<p>旋转矩阵：在乘以一个向量的时候改变向量的方向但不改变大小的效果并保持了手性的矩阵</p>
+<p>正规矩阵：<span class="math inline">\(A^{*}A=AA^*\)</span>，<span
+class="math inline">\(A^*\)</span>为<span
+class="math inline">\(A\)</span>的共轭矩阵</p>
+<p>共轭矩阵：<span class="math inline">\(A^*=(\bar{A})^T\)</span>，<span
+class="math inline">\(\bar{A}\)</span>表示矩阵的复共轭</p>
+<p>奇异矩阵：满足方阵，矩阵的行列式是否等于0，若等于0，称矩阵A为奇异矩阵</p>
+<p>非奇异矩阵：同上，若不等于0，称矩阵A为非奇异矩阵。</p>
+<p>稀疏矩阵：若数值为0的元素数目远远多于非0元素的数目，并且非0元素分布没有规律时</p>
+<p>稠密矩阵：与上相反，若非0元素数目占大多数时</p>
+<p>增广矩阵：扩增矩阵，在系数矩阵的右边添上一列，这一列是线性方程组的等号右边的值</p>
+<p>系数矩阵：将方程组的系数组成矩阵来计算方程的解</p>
+<p>相似矩阵：判断特征值、行列式、迹、秩是否相等，只作为判断矩阵是否相似的必要条件，而非充分条件。</p>
+]]></content>
+      <categories>
+        <category>Note</category>
       </categories>
       <tags>
-        <tag>Python</tag>
-        <tag>机器学习</tag>
-        <tag>神经网络</tag>
-        <tag>NumPy</tag>
+        <tag>数学</tag>
       </tags>
   </entry>
   <entry>
-    <title>神经网络笔记Ⅱ</title>
-    <url>/hexo_blog/2018/08/20/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A1/</url>
-    <content><![CDATA[<p>接 <a href="/hexo_blog/2018/04/14/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A0/" title="神经网络笔记Ⅰ">神经网络笔记Ⅰ</a></p>
-<p>从CNN到RNN再到LSTM，深入了解三种神经网络。其中，CNN主要用于图像识别与处理，RNN处理具有动态时序行为的任务，而LSTM则在此基础上添加了记忆单元，更适合处理时间序列中间隔和延迟非常长的事件。仅靠前向传播已不满足要求，反向传播也变得重要。</p>
+    <title>树莓派控制手机自动拍照</title>
+    <url>/hexo_blog/2023/11/26/IOT/%E6%A0%91%E8%8E%93%E6%B4%BE/%E6%A0%91%E8%8E%93%E6%B4%BE%E6%8E%A7%E5%88%B6%E6%89%8B%E6%9C%BA%E8%87%AA%E5%8A%A8%E6%8B%8D%E7%85%A7/</url>
+    <content><![CDATA[<p>树莓派使用 Python 控制手机上的 DroidCam 自动拍照</p>
 <span id="more"></span>
-<h2 id="cnn卷积神经网络">CNN（卷积神经网络）</h2>
-<p><a
-href="https://www.zybuluo.com/hanbingtao/note/485480">零基础入门深度学习(4)
-- 卷积神经网络</a></p>
-<p>CNN（Convolutional Neural
-Netword，卷积神经网络）是一种前馈神经网络，人工神经元可以响应周围单元，可以大型图像处理。卷积神经网络包括卷积层和池化层。</p>
-<p>基础结构</p>
+<h2 id="需求">需求</h2>
+<p>新买了个盆栽，想要关注盆栽的生长情况。如果是摄像，“代价”又有点大。想到了
+DroidCam
+这个软件，能够让手机变成监控摄像头。于是，可以利用它做一个自动拍照的延时摄影。</p>
+<h2 id="前期准备">前期准备</h2>
 <ul>
-<li>特征提取层，每个神经元的输入与前一层的局部接受域相连，并提取该局部的特征。一旦该局部特征被提取后，它与其他特征间的位置关系也随之确定下来</li>
-<li>特征映射层，网络的每个计算层由多个特征映射组成，每个特征映射是一个平面，平面上所有的神经元的权值相等。采用影响函数较小的<span
-class="math inline">\(sigmoid\)</span>函数作为卷积网络的激活函数，使得特征映射具有位移不变性</li>
+<li>手机（IOS、Android 均可） + DroidCam 软件</li>
+<li>树莓派（常年开机的小主机，能用 Python就行）</li>
+<li>其他重要的内容：
+<ul>
+<li>手机支架</li>
+<li>手机充电的电源</li>
+</ul></li>
 </ul>
-<p>CNN主要用来识别位移、缩放以及其他形式扭曲不变性的二维图形。由于CNN的特征检测层通过训练数据进行学习，所以在使用CNN时，避免了显示的特征抽取，从而隐式地从训练数据中进行学习。由于同一特征映射面上的神经元权值相同，所以网络可以并行学习。</p>
-<p>对于图像识别任务</p>
-<p>全连接网络</p>
-<ol type="1">
-<li>参数数量过多</li>
-<li>像素间的位置信息</li>
-<li>网络层数限制</li>
-</ol>
-<p>解决方案：</p>
-<ol type="1">
-<li>局部连接——不是全连接，减少参数数量</li>
-<li>权值共享——一组连接可以共享共一个权重，再一次减少参数数量</li>
-<li>下采样——使用池化（Pooling）来减少每层的样本数，进一步减少参数数量，提供<strong>鲁棒性
-</strong></li>
-</ol>
-<p><strong>P.S.</strong></p>
-<p>鲁棒是Robust的音译，也就是健壮和强壮的意思。它是在异常和危险情况下系统生存的关键。比如说，计算机软件在输入错误、磁盘故障、网络过载或有意攻击情况下，能否不死机、不崩溃，就是该软件的鲁棒性。所谓“鲁棒性”，是指控制系统在一定（结构，大小）的参数摄动下，维持其它某些性能的特性。根据对性能的不同定义，可分为稳定鲁棒性和性能鲁棒性。以闭环系统的鲁棒性作为目标设计得到的固定控制器称为鲁棒控制器。</p>
-<p>卷积神经网络：<strong>卷积层、Pooling层（池化层）、全连接层</strong></p>
-<p><code>INPUT -&gt; [[CONV]*N -&gt; POOL]*M -&gt; [FC]*K</code></p>
-<p>N个卷积层叠加再加上一个池化层，重复M次，加上K个全连接层。</p>
-<p>概念介绍和计算方式请看下面这篇博客</p>
-<p><a
-href="http://www.cnblogs.com/charlotte77/p/7759802.html">【深度学习系列】卷积神经网络CNN原理详解(一)——基本原理</a></p>
-<p>总结：</p>
-<ol type="1">
-<li>池化方式：
+<p>大致效果如图</p>
+<p><img
+src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/raspberry_droidcam/IMG_20201126_201708.jpg" /></p>
+<h2 id="v1.0">v1.0</h2>
+<p>在搭建完基本的设施之后，首先测试拍照功能，命令如下所示，将 $IP_ADDR
+换成手机 IP 地址即可，DroidCam 主界面也会显示。默认端口是 4747</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget <span class="string">&quot;http://<span class="variable">$IP_ADDR</span>:4747/cam/1/frame.jpg&quot;</span> -O $(<span class="built_in">date</span> +<span class="string">&quot;%s&quot;</span>).jpg</span><br></pre></td></tr></table></figure>
+<p>然而，在把这行命令放到树莓派上的定时任务中时，遇到了几个问题</p>
+<h3 id="晚上光线暗">晚上光线暗</h3>
+<p>打开手机摄像头的闪光灯，命令如下所示</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">curl <span class="string">&quot;http://<span class="variable">$IP_ADDR</span>:4747/cam/1/led_toggle&quot;</span> \</span><br><span class="line">  --compressed \</span><br><span class="line">  --insecure</span><br></pre></td></tr></table></figure>
+<h3 id="没有对焦">没有对焦</h3>
+<p>自动对焦命令如下所示</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">af_cmd = f<span class="string">&quot;&quot;</span><span class="string">&quot;curl &quot;</span>http://<span class="variable">$IP_ADDR</span>:4747/cam/1/af<span class="string">&quot; \</span></span><br><span class="line"><span class="string">  --compressed \</span></span><br><span class="line"><span class="string">  --insecure</span></span><br></pre></td></tr></table></figure>
+<p>这两个问题算是可以通过软件本身进行解决。但引发了一个新问题就是，不能知道手机是否已经开启闪光灯、是否已经对焦。前者可以通过时间判断，如果是白天则不开，但后者没办法控制。DroidCam
+的逻辑是，每次只能有一个客户端打开网址，如果有多个客户端打开，后者会把前者踢下线，且状态为止。因此，最后决定每次拍两张照片，用照片大小来判断图片质量。</p>
+<h3 id="droidcam-要求启动摄像头后才能拍照">DroidCam
+要求启动摄像头后，才能拍照</h3>
+<p>一个比较大的问题是，DroidCam
+要求启动摄像头后，才能拍照。而如果没有客户端打开网址，则摄像头不会启动。因此，需要在请求前，去开启网址。</p>
+<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">start_func</span>():</span><br><span class="line">    video_url = <span class="string">&quot;http://&#123;&#125;/video?640x480&quot;</span>.<span class="built_in">format</span>(ip_addr)</span><br><span class="line">    response = requests.get(video_url, stream=<span class="literal">True</span>)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;start_func&quot;</span>)</span><br></pre></td></tr></table></figure>
+<p>如果长时间没有启动
+DroidCam，手机则会自动息屏，导致之后的拍照全部失败。</p>
+<h2 id="实现">实现</h2>
+<p>见下面代码的 <code>take_picture.py</code>
+部分，只需将其设置为定时任务即可
+<code>*/10 * * * * cd /root/pics &amp;&amp; /usr/bin/python3 take_picture.py</code>。每
+10 分钟会去拍一次照，且会拍两张，以时间戳保存在 <code>/root/pics</code>
+目录下。</p>
+<script src="https://gist.github.com/wnma3mz/a583832cbca9965a733834fbcae42352.js"></script>
+<h2 id="v2.0">v2.0</h2>
+<p>照片数据问题解决了，接下来就要处理照片的删除、处理、合成逻辑了。</p>
+<p>见上面代码的 <code>process.py</code> 部分</p>
+<p>简单来说分为三个部分：</p>
 <ol type="1">
-<li>MaxPooling：取滑动窗口里最大的值</li>
-<li>AveragePooling：取滑动窗口内所有值的平均值</li>
-<li>StochasticPooling，介于两者之间，通过对像素点按照数值大小赋予概率，再按照概率进行亚采样；</li>
-</ol></li>
-<li>Zero Padding(补零)：保留原始图片的尺寸</li>
+<li>获取当前文件夹下所有的图片，以时间戳进行排序，时间戳间隔小于 60
+秒的照片，只保留一张（以图片大小定）。</li>
+<li>按照日期进行移动，并打包成 zip 文件。</li>
+<li>在 PC 端运行
+<code>python3 process.py --convert</code>，会拉取树莓派的压缩包，并解压
+zip 文件，最后合成视频。（为什么不在树莓派上，性能太弱，很慢）</li>
 </ol>
-<h4 id="卷积">卷积</h4>
-<p>对于一个<span class="math inline">\(l_1*l_2\)</span>的图像，使用<span
-class="math inline">\(f_1*f_2\)</span>的filter进行卷积（后者小于前者），得到一个<span
-class="math inline">\(f_3*f_4\)</span>的feature map。</p>
-<p><span class="math display">\[
-a_{ij}=f(\sum\limits_{m=0}^{f_1-1}\sum\limits_{n=0}^{f_2-1}w_{mn}x_{(m+i)(n+j)}+w_b)
-\]</span></p>
-<p><span class="math inline">\(a_{ij}\)</span>表示的是feature
-map的元素，<span
-class="math inline">\(w_{mn}\)</span>表示对应元素，<span
-class="math inline">\(f\)</span>函数表示激活函数，<span
-class="math inline">\(x_{(m+i)(n+j)}\)</span>表示图像的元素，<span
-class="math inline">\(w_b\)</span>表示偏置项，这里从0开始计数，所以需要-1。</p>
-<p>这里简单起见，步幅设置为1。</p>
-<p>图像大小、步幅和卷积后的Feature Map大小是有关系的。</p>
-<p><span class="math inline">\(W_2=(W_1-F+2P)/S+1\)</span></p>
-<p><span class="math inline">\(H_2=(H_1-F+2P)/S+1\)</span></p>
-<p><span class="math inline">\(W_2\)</span>是卷积后Feature
-Map的宽度；<span
-class="math inline">\(W_1\)</span>是卷积前图像的宽度；<span
-class="math inline">\(F\)</span>是filter的宽度；<span
-class="math inline">\(P\)</span>是<strong>Zero
-Padding</strong>数量，<strong>Zero
-Padding</strong>是指在原始图像周围补几圈0，如果的值是1，那么就补1圈0；<span
-class="math inline">\(S\)</span>是<strong>步幅</strong>；<span
-class="math inline">\(H_2\)</span>是卷积后Feature Map的高度；<span
-class="math inline">\(H_1\)</span>是卷积前图像的宽度。</p>
-<p>二维卷积公式，已知矩阵<span class="math inline">\(A\)</span>、<span
-class="math inline">\(B\)</span>，行列数分别用<span
-class="math inline">\(m\)</span>和<span
-class="math inline">\(n\)</span>表示。卷积公式如下</p>
-<p><span
-class="math inline">\(C_{st}=\sum\limits_0^{m_A-1}\sum\limits_0^{n_A-1}A_{mn}B_{(s-m)(t-n)}\)</span></p>
-<p><span class="math inline">\(0\le s&lt;m_A+m_B-1\)</span></p>
-<p><span class="math inline">\(0\le t&lt;n_A+n_B-1\)</span></p>
-<p>也可以写成<span class="math inline">\(C=A*B\)</span></p>
-<p>数学中的卷积与卷积神经网络中的卷积是一定区别的。可以把神经网络中的卷积称为<strong>互相关(cross-correlation)</strong>。这两种操作转换方式，可以通过把矩阵<span
-class="math inline">\(A\)</span>翻转180度，再交换<span
-class="math inline">\(A\)</span>、<span
-class="math inline">\(B\)</span>运算位置，即可以将卷积变为互相关。</p>
-<h2 id="rnn循环神经网络">RNN（循环神经网络）</h2>
-<p>RNN（Recurrent neural
-Network，循环神经网络/多层反馈网络）是一种节点定向连接成环的人工神经网络。这种网络的内部状态可以展示动态时序行为。不同于前馈神经网络的是，RNN可以利用它内部的记忆来处理任意时序的输入序列，这让它可以更容易处理，如不分段的手写识别、语音识别等。</p>
-<p>RNN网络的本质特征是处理单元之间既有内部的反馈连接又有前馈连接。从系统的观点看，它是一个反馈动力系统，在计算过程中体现过程动态特性，比前馈神经网络具有更强的动态行为和计算能力。</p>
-<p><a
-href="https://iamtrask.github.io/2015/11/15/anyone-can-code-lstm/">参考文章</a></p>
-<p><strong><em>Talk</em> <em>is</em> cheap. <em>Show</em> <em>me</em>
-<em>the</em> <em>code</em>.</strong></p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="comment"># coding: utf-8</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">import</span> copy</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">sigmoid</span>(<span class="params">x, deriv=<span class="literal">False</span></span>):</span><br><span class="line">    <span class="comment"># sigmoid激活函数</span></span><br><span class="line">    <span class="keyword">if</span> (deriv == <span class="literal">True</span>):</span><br><span class="line">        <span class="keyword">return</span> x * (<span class="number">1</span> - x)</span><br><span class="line">    <span class="keyword">return</span> <span class="number">1</span> / (<span class="number">1</span> + np.exp(-x))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 存储10进制转2进制的字典</span></span><br><span class="line">int2binary = &#123;&#125;</span><br><span class="line"><span class="comment"># 二进制数最大长度</span></span><br><span class="line">binary_dim = <span class="number">8</span></span><br><span class="line"><span class="comment"># 最大10进制数</span></span><br><span class="line">larget_number = <span class="number">2</span> ** binary_dim</span><br><span class="line">binary = np.unpackbits(</span><br><span class="line">    np.array([<span class="built_in">range</span>(larget_number)], dtype=np.uint8).T, axis=<span class="number">1</span>)</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(larget_number):</span><br><span class="line">    int2binary[i] = binary[i]</span><br><span class="line"></span><br><span class="line"><span class="comment"># 梯度</span></span><br><span class="line">alpha = <span class="number">0.1</span></span><br><span class="line"><span class="comment"># 输入维度，两个数</span></span><br><span class="line">input_dim = <span class="number">2</span></span><br><span class="line"><span class="comment"># 隐单元维度</span></span><br><span class="line">hidden_dim = <span class="number">16</span></span><br><span class="line"><span class="comment"># 输出维度，一个数</span></span><br><span class="line">output_dim = <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 两层神经网络。每层权重，[-1, 1]</span></span><br><span class="line">syn_0 = <span class="number">2</span> * np.random.random((input_dim, hidden_dim)) - <span class="number">1</span></span><br><span class="line">syn_1 = <span class="number">2</span> * np.random.random((hidden_dim, output_dim)) - <span class="number">1</span></span><br><span class="line"><span class="comment"># 神经网络连接前一个隐藏层的权重</span></span><br><span class="line">syn_h = <span class="number">2</span> * np.random.random((hidden_dim, hidden_dim)) - <span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 存储权重的更新</span></span><br><span class="line">syn_0_update = np.zeros_like(syn_0)</span><br><span class="line">syn_1_update = np.zeros_like(syn_1)</span><br><span class="line">syn_h_update = np.zeros_like(syn_h)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> _ <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">10000</span>):</span><br><span class="line">    <span class="comment"># 一个简单的加法问题，c=a+b</span></span><br><span class="line">    <span class="comment"># 在0，128（避免相加超过256）之间一个随机数a，十进制表示</span></span><br><span class="line">    a_int = np.random.randint(larget_number / <span class="number">2</span>)</span><br><span class="line">    <span class="comment"># 二进制表示a</span></span><br><span class="line">    a = int2binary[a_int]</span><br><span class="line"></span><br><span class="line">    b_int = np.random.randint(larget_number / <span class="number">2</span>)</span><br><span class="line">    b = int2binary[b_int]</span><br><span class="line">    <span class="comment"># 正确答案c</span></span><br><span class="line">    c_int = a_int + b_int</span><br><span class="line">    c = int2binary[c_int]</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 神经网络计算的答案</span></span><br><span class="line">    d = np.zeros_like(c)</span><br><span class="line">    <span class="comment"># 计算的误差</span></span><br><span class="line">    overallError = <span class="number">0</span></span><br><span class="line">    <span class="comment"># 记录l2_delta、l1</span></span><br><span class="line">    layer_2_deltas = []</span><br><span class="line">    layer_1_values = []</span><br><span class="line">    <span class="comment"># l1初始值全为0</span></span><br><span class="line">    layer_1_values.append(np.zeros(hidden_dim))</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 前向传播。特征是二进制数维度。</span></span><br><span class="line">    <span class="keyword">for</span> position <span class="keyword">in</span> <span class="built_in">range</span>(binary_dim):</span><br><span class="line">        <span class="comment"># X是1*2的矩阵，[[m, n]]，m、n是二进制数a、b在每个位置的01值。从最后一维开始遍历</span></span><br><span class="line">        X = np.array(</span><br><span class="line">            [[a[binary_dim - position - <span class="number">1</span>], b[binary_dim - position - <span class="number">1</span>]]])</span><br><span class="line">        <span class="comment"># y是1*1的矩阵。[[t]]，t是二进制数c在每个位置的01值</span></span><br><span class="line">        y = np.array([[c[binary_dim - position - <span class="number">1</span>]]]).T</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 计算输出层，np.dot(X,syn_0)是正常操作:从输入层到隐藏层;np.dot(layer_1_values[-1],syn_h):从前一层隐藏层到这一层隐藏层</span></span><br><span class="line">        layer_1 = sigmoid(np.dot(X, syn_0) + np.dot(layer_1_values[-<span class="number">1</span>], syn_h))</span><br><span class="line">        layer_2 = sigmoid(np.dot(layer_1, syn_1))</span><br><span class="line">        <span class="comment"># 计算l2偏差</span></span><br><span class="line">        layer_2_error = y - layer_2</span><br><span class="line">        <span class="comment"># 保存偏差加权导数l2_delta</span></span><br><span class="line">        layer_2_deltas.append(layer_2_error * sigmoid(layer_2, deriv=<span class="literal">True</span>))</span><br><span class="line">        <span class="comment"># 误差总和</span></span><br><span class="line">        overallError += np.<span class="built_in">abs</span>(layer_2_error[<span class="number">0</span>])</span><br><span class="line">        <span class="comment"># 神经网络每个维度的计算的01值</span></span><br><span class="line">        d[binary_dim - position - <span class="number">1</span>] = np.<span class="built_in">round</span>(layer_2[<span class="number">0</span>][<span class="number">0</span>])</span><br><span class="line">        <span class="comment"># 保存l1层，在下次迭代时计算下次传播的隐藏层</span></span><br><span class="line">        <span class="comment"># layer_1_values.append(copy.deepcopy(layer_1))</span></span><br><span class="line">        layer_1_values.append(layer_1.copy())</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 存储反向传播的l1_delta</span></span><br><span class="line">    futuer_layer_1_delta = np.zeros(hidden_dim)</span><br><span class="line">    <span class="comment"># 反向传播.(input + prev_input) -&gt; hidden -&gt; output</span></span><br><span class="line">    <span class="keyword">for</span> position <span class="keyword">in</span> <span class="built_in">range</span>(binary_dim):</span><br><span class="line">        <span class="comment"># 从第一维开始遍历</span></span><br><span class="line">        X = np.array([[a[position], b[position]]])</span><br><span class="line">        <span class="comment"># 选出前面计算得到的l1作为当前层，反向计算</span></span><br><span class="line">        layer_1 = layer_1_values[-position - <span class="number">1</span>]</span><br><span class="line">        <span class="comment"># 选择l1前一层的隐藏层</span></span><br><span class="line">        prev_layer_1 = layer_1_values[-position - <span class="number">2</span>]</span><br><span class="line">        <span class="comment"># 选择出l2，当前层输出的误差</span></span><br><span class="line">        layer_2_delta = layer_2_deltas[-position - <span class="number">1</span>]</span><br><span class="line">        <span class="comment"># 计算当前隐藏层误差，计算前一层的l1_delta到当前层，加上正常操作。</span></span><br><span class="line">        layer_1_delta = (np.dot(futuer_layer_1_delta, syn_h.T) +</span><br><span class="line">                         layer_2_delta.dot(syn_1.T)) * sigmoid(layer_1, deriv=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 更新权重更新的参数。将l1转换为二维矩阵</span></span><br><span class="line">        syn_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)</span><br><span class="line">        syn_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)</span><br><span class="line">        <span class="comment"># X已经是二维矩阵，不需要转换</span></span><br><span class="line">        syn_0_update += X.T.dot(layer_1_delta)</span><br><span class="line">        <span class="comment"># 替换futuer_l1</span></span><br><span class="line">        futuer_layer_1_delta = layer_1_delta</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 调整每层权重</span></span><br><span class="line">    syn_0 += syn_0_update * alpha</span><br><span class="line">    syn_1 += syn_1_update * alpha</span><br><span class="line">    syn_h += syn_h_update * alpha</span><br><span class="line">    <span class="comment"># 归零更新权重参数</span></span><br><span class="line">    syn_0_update *= <span class="number">0</span></span><br><span class="line">    syn_1_update *= <span class="number">0</span></span><br><span class="line">    syn_h_update *= <span class="number">0</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> (_ % <span class="number">1000</span> == <span class="number">0</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Error: &quot;</span> + <span class="built_in">str</span>(overallError))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;Pred: &quot;</span> + <span class="built_in">str</span>(d))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;True: &quot;</span> + <span class="built_in">str</span>(c))</span><br><span class="line">        <span class="comment"># 二进制转十进制</span></span><br><span class="line">        out = <span class="number">0</span></span><br><span class="line">        <span class="keyword">for</span> index, x <span class="keyword">in</span> <span class="built_in">enumerate</span>(<span class="built_in">reversed</span>(d)):</span><br><span class="line">            out += x * (<span class="number">2</span>**index)</span><br><span class="line">        <span class="built_in">print</span>(<span class="built_in">str</span>(a_int) + <span class="string">&quot; + &quot;</span> + <span class="built_in">str</span>(b_int) + <span class="string">&quot;=&quot;</span> + <span class="built_in">str</span>(out))</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;-------------&quot;</span>)</span><br></pre></td></tr></table></figure>
-<p>将前向传播比喻为字母表顺序读出，反向传播就是字母表反序读出。</p>
-<h2 id="lstm时间递归神经网络">LSTM（时间递归神经网络）</h2>
-<p>长短期记忆（英语：Long Short-Term
-Memory，LSTM）是一种时间递归神经网络（RNN），论文首次发表于1997年。由于独特的设计结构，LSTM适合于处理和预测时间序列中间隔和延迟非常长的重要事件。</p>
-<p>LSTM的表现通常比时间递归神经网络及隐马尔科夫模型（HMM）更好，比如用在不分段连续手写识别上。作为非线性模型，LSTM可作为复杂的非线性单元用于构造更大型深度神经网络。</p>
+<p>ffmpeg 合并命令如下所示，会预先按照时间戳排序，写入到
+<code>input.txt</code> 文件中，然后合并成视频。</p>
+<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">ffmpeg -f concat -i input.txt -vf <span class="string">&quot;fps=25&quot;</span> <span class="variable">$OUTPUT</span></span><br></pre></td></tr></table></figure>
+<h2 id="成品">成品</h2>
+<video id="video" controls="" preload="none">
+<source id="mp4" src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/raspberry_droidcam/301_1697855508.mp4" type="video/mp4">
+<p></videos></p>
 ]]></content>
       <categories>
-        <category>DeepLearning</category>
+        <category>Raspberry</category>
       </categories>
       <tags>
         <tag>Python</tag>
-        <tag>神经网络</tag>
-        <tag>NumPy</tag>
+        <tag>DroidCam</tag>
       </tags>
   </entry>
   <entry>
@@ -8780,81 +8929,4 @@ https://A/ics）。这样就完成了所有的设置</p>
         <tag>Linux</tag>
       </tags>
   </entry>
-  <entry>
-    <title>树莓派控制手机自动拍照</title>
-    <url>/hexo_blog/2023/11/26/IOT/%E6%A0%91%E8%8E%93%E6%B4%BE/%E6%A0%91%E8%8E%93%E6%B4%BE%E6%8E%A7%E5%88%B6%E6%89%8B%E6%9C%BA%E8%87%AA%E5%8A%A8%E6%8B%8D%E7%85%A7/</url>
-    <content><![CDATA[<p>树莓派使用 Python 控制手机上的 DroidCam 自动拍照</p>
-<span id="more"></span>
-<h2 id="需求">需求</h2>
-<p>新买了个盆栽，想要关注盆栽的生长情况。如果是摄像，“代价”又有点大。想到了
-DroidCam
-这个软件，能够让手机变成监控摄像头。于是，可以利用它做一个自动拍照的延时摄影。</p>
-<h2 id="前期准备">前期准备</h2>
-<ul>
-<li>手机（IOS、Android 均可） + DroidCam 软件</li>
-<li>树莓派（常年开机的小主机，能用 Python就行）</li>
-<li>其他重要的内容：
-<ul>
-<li>手机支架</li>
-<li>手机充电的电源</li>
-</ul></li>
-</ul>
-<p>大致效果如图</p>
-<p><img
-src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/raspberry_droidcam/IMG_20201126_201708.jpg" /></p>
-<h2 id="v1.0">v1.0</h2>
-<p>在搭建完基本的设施之后，首先测试拍照功能，命令如下所示，将 $IP_ADDR
-换成手机 IP 地址即可，DroidCam 主界面也会显示。默认端口是 4747</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">wget <span class="string">&quot;http://<span class="variable">$IP_ADDR</span>:4747/cam/1/frame.jpg&quot;</span> -O $(<span class="built_in">date</span> +<span class="string">&quot;%s&quot;</span>).jpg</span><br></pre></td></tr></table></figure>
-<p>然而，在把这行命令放到树莓派上的定时任务中时，遇到了几个问题</p>
-<h3 id="晚上光线暗">晚上光线暗</h3>
-<p>打开手机摄像头的闪光灯，命令如下所示</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">curl <span class="string">&quot;http://<span class="variable">$IP_ADDR</span>:4747/cam/1/led_toggle&quot;</span> \</span><br><span class="line">  --compressed \</span><br><span class="line">  --insecure</span><br></pre></td></tr></table></figure>
-<h3 id="没有对焦">没有对焦</h3>
-<p>自动对焦命令如下所示</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">af_cmd = f<span class="string">&quot;&quot;</span><span class="string">&quot;curl &quot;</span>http://<span class="variable">$IP_ADDR</span>:4747/cam/1/af<span class="string">&quot; \</span></span><br><span class="line"><span class="string">  --compressed \</span></span><br><span class="line"><span class="string">  --insecure</span></span><br></pre></td></tr></table></figure>
-<p>这两个问题算是可以通过软件本身进行解决。但引发了一个新问题就是，不能知道手机是否已经开启闪光灯、是否已经对焦。前者可以通过时间判断，如果是白天则不开，但后者没办法控制。DroidCam
-的逻辑是，每次只能有一个客户端打开网址，如果有多个客户端打开，后者会把前者踢下线，且状态为止。因此，最后决定每次拍两张照片，用照片大小来判断图片质量。</p>
-<h3 id="droidcam-要求启动摄像头后才能拍照">DroidCam
-要求启动摄像头后，才能拍照</h3>
-<p>一个比较大的问题是，DroidCam
-要求启动摄像头后，才能拍照。而如果没有客户端打开网址，则摄像头不会启动。因此，需要在请求前，去开启网址。</p>
-<figure class="highlight python"><table><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">start_func</span>():</span><br><span class="line">    video_url = <span class="string">&quot;http://&#123;&#125;/video?640x480&quot;</span>.<span class="built_in">format</span>(ip_addr)</span><br><span class="line">    response = requests.get(video_url, stream=<span class="literal">True</span>)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;start_func&quot;</span>)</span><br></pre></td></tr></table></figure>
-<p>如果长时间没有启动
-DroidCam，手机则会自动息屏，导致之后的拍照全部失败。</p>
-<h2 id="实现">实现</h2>
-<p>见下面代码的 <code>take_picture.py</code>
-部分，只需将其设置为定时任务即可
-<code>*/10 * * * * cd /root/pics &amp;&amp; /usr/bin/python3 take_picture.py</code>。每
-10 分钟会去拍一次照，且会拍两张，以时间戳保存在 <code>/root/pics</code>
-目录下。</p>
-<script src="https://gist.github.com/wnma3mz/a583832cbca9965a733834fbcae42352.js"></script>
-<h2 id="v2.0">v2.0</h2>
-<p>照片数据问题解决了，接下来就要处理照片的删除、处理、合成逻辑了。</p>
-<p>见上面代码的 <code>process.py</code> 部分</p>
-<p>简单来说分为三个部分：</p>
-<ol type="1">
-<li>获取当前文件夹下所有的图片，以时间戳进行排序，时间戳间隔小于 60
-秒的照片，只保留一张（以图片大小定）。</li>
-<li>按照日期进行移动，并打包成 zip 文件。</li>
-<li>在 PC 端运行
-<code>python3 process.py --convert</code>，会拉取树莓派的压缩包，并解压
-zip 文件，最后合成视频。（为什么不在树莓派上，性能太弱，很慢）</li>
-</ol>
-<p>ffmpeg 合并命令如下所示，会预先按照时间戳排序，写入到
-<code>input.txt</code> 文件中，然后合并成视频。</p>
-<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">ffmpeg -f concat -i input.txt -vf <span class="string">&quot;fps=25&quot;</span> <span class="variable">$OUTPUT</span></span><br></pre></td></tr></table></figure>
-<h2 id="成品">成品</h2>
-<video id="video" controls="" preload="none">
-<source id="mp4" src="https://raw.githubusercontent.com/wnma3mz/blog_posts/master/imgs/raspberry_droidcam/301_1697855508.mp4" type="video/mp4">
-<p></videos></p>
-]]></content>
-      <categories>
-        <category>Raspberry</category>
-      </categories>
-      <tags>
-        <tag>Python</tag>
-        <tag>DroidCam</tag>
-      </tags>
-  </entry>
 </search>
diff --git a/sitemap.xml b/sitemap.xml
index be8f84bb..7af1beda 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,7 +2,7 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   
   <url>
-    <loc>https://wnma3mz.github.io/2024/08/08/Knowledge-Distillation-in-LLM/</loc>
+    <loc>https://wnma3mz.github.io/2022/12/31/ShortPaperReading/</loc>
     
     <lastmod>2024-09-14</lastmod>
     
@@ -11,7 +11,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2022/12/31/ShortPaperReading/</loc>
+    <loc>https://wnma3mz.github.io/2024/08/08/Knowledge-Distillation-in-LLM/</loc>
     
     <lastmod>2024-09-14</lastmod>
     
@@ -74,7 +74,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/04/14/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A0/</loc>
+    <loc>https://wnma3mz.github.io/2018/08/20/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A1/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -83,7 +83,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/08/20/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A1/</loc>
+    <loc>https://wnma3mz.github.io/2018/04/14/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E7%AC%94%E8%AE%B0%E2%85%A0/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -101,7 +101,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/</loc>
+    <loc>https://wnma3mz.github.io/2023/08/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E6%A8%A1%E4%BB%BF%E6%B8%B8%E6%88%8F%E2%80%94%E2%80%94%E6%9C%BA%E5%99%A8%E4%BC%9A%E6%80%9D%E8%80%83%E5%90%97/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -110,7 +110,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/08/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/%E6%A8%A1%E4%BB%BF%E6%B8%B8%E6%88%8F%E2%80%94%E2%80%94%E6%9C%BA%E5%99%A8%E4%BC%9A%E6%80%9D%E8%80%83%E5%90%97/</loc>
+    <loc>https://wnma3mz.github.io/2018/08/05/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A2/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -119,7 +119,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/</loc>
+    <loc>https://wnma3mz.github.io/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -128,7 +128,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/05/23/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A1/</loc>
+    <loc>https://wnma3mz.github.io/2018/05/13/%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0/NLTK%E9%98%85%E8%AF%BB%E7%AC%94%E8%AE%B0%E2%85%A0/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -254,7 +254,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/</loc>
+    <loc>https://wnma3mz.github.io/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -263,7 +263,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2017/11/24/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BB%8E%E5%AE%89%E8%A3%85%E5%88%B0%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C/</loc>
+    <loc>https://wnma3mz.github.io/2021/07/21/%E6%95%B0%E6%8D%AE%E5%BA%93/Mongodb%E4%BA%8C/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -335,7 +335,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/08/28/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E5%AD%97%E7%AC%A6%E4%B8%B2/</loc>
+    <loc>https://wnma3mz.github.io/2018/04/05/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E6%B5%B7%E9%87%8F%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E9%97%AE%E9%A2%98%EF%BC%88HashMap+%E5%88%86%E6%B2%BB%E6%B3%95%E3%80%81%E4%BD%8D%E5%9B%BE%E6%B3%95%EF%BC%89/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -344,7 +344,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/04/05/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E6%B5%B7%E9%87%8F%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E9%97%AE%E9%A2%98%EF%BC%88HashMap+%E5%88%86%E6%B2%BB%E6%B3%95%E3%80%81%E4%BD%8D%E5%9B%BE%E6%B3%95%EF%BC%89/</loc>
+    <loc>https://wnma3mz.github.io/2023/08/28/%E4%BC%A0%E7%BB%9F%E7%AE%97%E6%B3%95/%E5%AD%97%E7%AC%A6%E4%B8%B2/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -380,7 +380,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2020/09/16/ipv6%E5%85%8D%E6%B5%81/</loc>
+    <loc>https://wnma3mz.github.io/2023/11/17/Tracing-Model-Outputs-to-the-Training-Data/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -389,7 +389,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/11/17/Tracing-Model-Outputs-to-the-Training-Data/</loc>
+    <loc>https://wnma3mz.github.io/2020/09/16/ipv6%E5%85%8D%E6%B5%81/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -443,7 +443,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/03/14/Python/Python%E9%AD%94%E6%9C%AF%E6%96%B9%E6%B3%95/</loc>
+    <loc>https://wnma3mz.github.io/2023/11/01/Python/%E5%A4%9A%E8%BF%9B%E7%A8%8B%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F%E9%97%AE%E9%A2%98%E8%AE%B0%E5%BD%95/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -452,7 +452,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/11/01/Python/%E5%A4%9A%E8%BF%9B%E7%A8%8B%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F%E9%97%AE%E9%A2%98%E8%AE%B0%E5%BD%95/</loc>
+    <loc>https://wnma3mz.github.io/2018/03/14/Python/Python%E9%AD%94%E6%9C%AF%E6%96%B9%E6%B3%95/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -506,7 +506,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2021/02/03/PyTorch/torch-scheduler%E8%AE%B0%E5%BD%95/</loc>
+    <loc>https://wnma3mz.github.io/2023/09/03/PyTorch/%E4%BA%A4%E5%8F%89%E7%86%B5%E7%9A%84%E5%8F%8D%E5%90%91%E4%BC%A0%E6%92%AD%20Step%20by%20Step/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -515,7 +515,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/09/03/PyTorch/%E4%BA%A4%E5%8F%89%E7%86%B5%E7%9A%84%E5%8F%8D%E5%90%91%E4%BC%A0%E6%92%AD%20Step%20by%20Step/</loc>
+    <loc>https://wnma3mz.github.io/2021/02/03/PyTorch/torch-scheduler%E8%AE%B0%E5%BD%95/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -524,7 +524,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/12/03/PyTorch/pytorch_cpp_extension/</loc>
+    <loc>https://wnma3mz.github.io/2018/01/25/IOT/%E7%94%A8ADB%E8%B0%83%E8%AF%95%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -533,7 +533,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2018/01/25/IOT/%E7%94%A8ADB%E8%B0%83%E8%AF%95%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA/</loc>
+    <loc>https://wnma3mz.github.io/2023/12/03/PyTorch/pytorch_cpp_extension/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -551,7 +551,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/11/26/IOT/%E6%A0%91%E8%8E%93%E6%B4%BE/%E6%A0%91%E8%8E%93%E6%B4%BE%E6%8A%98%E8%85%BE%E8%AE%B0%E5%BD%95/</loc>
+    <loc>https://wnma3mz.github.io/2023/11/26/IOT/%E6%A0%91%E8%8E%93%E6%B4%BE/%E6%A0%91%E8%8E%93%E6%B4%BE%E6%8E%A7%E5%88%B6%E6%89%8B%E6%9C%BA%E8%87%AA%E5%8A%A8%E6%8B%8D%E7%85%A7/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -560,7 +560,7 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/2023/11/26/IOT/%E6%A0%91%E8%8E%93%E6%B4%BE/%E6%A0%91%E8%8E%93%E6%B4%BE%E6%8E%A7%E5%88%B6%E6%89%8B%E6%9C%BA%E8%87%AA%E5%8A%A8%E6%8B%8D%E7%85%A7/</loc>
+    <loc>https://wnma3mz.github.io/2023/11/26/IOT/%E6%A0%91%E8%8E%93%E6%B4%BE/%E6%A0%91%E8%8E%93%E6%B4%BE%E6%8A%98%E8%85%BE%E8%AE%B0%E5%BD%95/</loc>
     
     <lastmod>2024-06-15</lastmod>
     
@@ -641,56 +641,56 @@
 
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Python/</loc>
+    <loc>https://wnma3mz.github.io/tags/NLP/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/ICS/</loc>
+    <loc>https://wnma3mz.github.io/tags/LLM/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/NLP/</loc>
+    <loc>https://wnma3mz.github.io/tags/knowledge-distillation/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/LLM/</loc>
+    <loc>https://wnma3mz.github.io/tags/Python/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Raspberry/</loc>
+    <loc>https://wnma3mz.github.io/tags/ICS/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Inference/</loc>
+    <loc>https://wnma3mz.github.io/tags/Raspberry/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Math/</loc>
+    <loc>https://wnma3mz.github.io/tags/Inference/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/knowledge-distillation/</loc>
+    <loc>https://wnma3mz.github.io/tags/Math/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -704,42 +704,42 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/MoE/</loc>
+    <loc>https://wnma3mz.github.io/tags/Algorithm/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Loss/</loc>
+    <loc>https://wnma3mz.github.io/tags/Linux/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Algorithm/</loc>
+    <loc>https://wnma3mz.github.io/tags/Tmux/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Federated-Learning/</loc>
+    <loc>https://wnma3mz.github.io/tags/MoE/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Linux/</loc>
+    <loc>https://wnma3mz.github.io/tags/Loss/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Tmux/</loc>
+    <loc>https://wnma3mz.github.io/tags/Federated-Learning/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -753,14 +753,14 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/ipv6/</loc>
+    <loc>https://wnma3mz.github.io/tags/Interpretability/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Interpretability/</loc>
+    <loc>https://wnma3mz.github.io/tags/ipv6/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -781,42 +781,42 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Note/</loc>
+    <loc>https://wnma3mz.github.io/tags/Tailscale/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Tailscale/</loc>
+    <loc>https://wnma3mz.github.io/tags/Note/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E7%88%AC%E8%99%AB/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E6%A0%91%E8%8E%93%E6%B4%BE/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%BE%AE%E4%BF%A1/</loc>
+    <loc>https://wnma3mz.github.io/tags/Google-Charts/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E6%A0%91%E8%8E%93%E6%B4%BE/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E7%88%AC%E8%99%AB/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Google-Charts/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%BE%AE%E4%BF%A1/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -837,112 +837,112 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Matplotlib/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E8%B7%AF%E7%94%B1%E5%99%A8/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Numpy/</loc>
+    <loc>https://wnma3mz.github.io/tags/OpenWrt/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/pandas/</loc>
+    <loc>https://wnma3mz.github.io/tags/Android/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/PyTorch/</loc>
+    <loc>https://wnma3mz.github.io/tags/Termux/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Cpp/</loc>
+    <loc>https://wnma3mz.github.io/tags/Windows/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/scheduler/</loc>
+    <loc>https://wnma3mz.github.io/tags/IOS/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E6%95%B0%E5%AD%A6/</loc>
+    <loc>https://wnma3mz.github.io/tags/Kindle/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E4%BA%A4%E5%8F%89%E7%86%B5/</loc>
+    <loc>https://wnma3mz.github.io/tags/ADB/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E8%B7%AF%E7%94%B1%E5%99%A8/</loc>
+    <loc>https://wnma3mz.github.io/tags/Matplotlib/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/OpenWrt/</loc>
+    <loc>https://wnma3mz.github.io/tags/pandas/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Android/</loc>
+    <loc>https://wnma3mz.github.io/tags/Numpy/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Termux/</loc>
+    <loc>https://wnma3mz.github.io/tags/PyTorch/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Windows/</loc>
+    <loc>https://wnma3mz.github.io/tags/Cpp/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/IOS/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E6%95%B0%E5%AD%A6/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Kindle/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E4%BA%A4%E5%8F%89%E7%86%B5/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/ADB/</loc>
+    <loc>https://wnma3mz.github.io/tags/scheduler/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -998,259 +998,259 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Magic-Methods/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%A4%9A%E8%BF%9B%E7%A8%8B/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%A4%9A%E8%BF%9B%E7%A8%8B/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%86%85%E5%AD%98%E6%B3%84%E6%BC%8F/</loc>
+    <loc>https://wnma3mz.github.io/tags/HashMap/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%AD%97%E7%AC%A6%E4%B8%B2/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E4%BD%8D%E5%9B%BE%E6%B3%95/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/HashMap/</loc>
+    <loc>https://wnma3mz.github.io/tags/mongodb/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E4%BD%8D%E5%9B%BE%E6%B3%95/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E6%95%B0%E6%8D%AE%E5%BA%93/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/mongodb/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%AD%97%E7%AC%A6%E4%B8%B2/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E6%95%B0%E6%8D%AE%E5%BA%93/</loc>
+    <loc>https://wnma3mz.github.io/tags/Magic-Methods/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/SQL/</loc>
+    <loc>https://wnma3mz.github.io/tags/crontab/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Hadoop/</loc>
+    <loc>https://wnma3mz.github.io/tags/SQL/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E7%AB%AF%E5%8F%A3%E8%BD%AC%E5%8F%91/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Ubuntu/</loc>
+    <loc>https://wnma3mz.github.io/tags/ssh/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Tensorflow/</loc>
+    <loc>https://wnma3mz.github.io/tags/iptables/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/CUDNN/</loc>
+    <loc>https://wnma3mz.github.io/tags/firewall/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Hive/</loc>
+    <loc>https://wnma3mz.github.io/tags/ufw/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/crontab/</loc>
+    <loc>https://wnma3mz.github.io/tags/netsh/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Caffe/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%86%85%E7%BD%91%E7%A9%BF%E9%80%8F/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E9%A3%8E%E6%A0%BC%E8%BF%81%E7%A7%BB/</loc>
+    <loc>https://wnma3mz.github.io/tags/Ubuntu/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E7%AB%AF%E5%8F%A3%E8%BD%AC%E5%8F%91/</loc>
+    <loc>https://wnma3mz.github.io/tags/Tensorflow/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/ssh/</loc>
+    <loc>https://wnma3mz.github.io/tags/CUDNN/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/iptables/</loc>
+    <loc>https://wnma3mz.github.io/tags/Hadoop/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/firewall/</loc>
+    <loc>https://wnma3mz.github.io/tags/Hive/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/ufw/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/netsh/</loc>
+    <loc>https://wnma3mz.github.io/tags/Nginx/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E5%86%85%E7%BD%91%E7%A9%BF%E9%80%8F/</loc>
+    <loc>https://wnma3mz.github.io/tags/Github/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Nginx/</loc>
+    <loc>https://wnma3mz.github.io/tags/Webhook/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Github/</loc>
+    <loc>https://wnma3mz.github.io/tags/HTTPS/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Webhook/</loc>
+    <loc>https://wnma3mz.github.io/tags/Caffe/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/HTTPS/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E9%A3%8E%E6%A0%BC%E8%BF%81%E7%A7%BB/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/Optimization/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E7%AC%94%E8%AE%B0/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E6%8B%89%E6%A0%BC%E6%9C%97%E6%97%A5%E4%B9%98%E5%AD%90%E6%B3%95/</loc>
+    <loc>https://wnma3mz.github.io/tags/NLTK/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3/</loc>
+    <loc>https://wnma3mz.github.io/tags/Optimization/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E7%AC%94%E8%AE%B0/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/NLTK/</loc>
+    <loc>https://wnma3mz.github.io/tags/NumPy/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E6%8B%89%E6%A0%BC%E6%9C%97%E6%97%A5%E4%B9%98%E5%AD%90%E6%B3%95/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E7%9F%A9%E9%98%B5%E5%88%86%E8%A7%A3/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/tags/NumPy/</loc>
+    <loc>https://wnma3mz.github.io/tags/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -1266,14 +1266,14 @@
 
   
   <url>
-    <loc>https://wnma3mz.github.io/categories/Note/</loc>
+    <loc>https://wnma3mz.github.io/categories/PaperReading/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/categories/PaperReading/</loc>
+    <loc>https://wnma3mz.github.io/categories/Note/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -1322,14 +1322,14 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/categories/Scrapy/</loc>
+    <loc>https://wnma3mz.github.io/categories/Visualization/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/categories/Visualization/</loc>
+    <loc>https://wnma3mz.github.io/categories/Scrapy/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
@@ -1350,14 +1350,14 @@
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/categories/Algorithm/</loc>
+    <loc>https://wnma3mz.github.io/categories/Android/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
   </url>
   
   <url>
-    <loc>https://wnma3mz.github.io/categories/Android/</loc>
+    <loc>https://wnma3mz.github.io/categories/Algorithm/</loc>
     <lastmod>2024-09-14</lastmod>
     <changefreq>weekly</changefreq>
     <priority>0.2</priority>
