-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
username
committed
May 13, 2024
1 parent
7b10017
commit 39c899e
Showing
32 changed files
with
1,008 additions
and
143 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,283 @@ | ||
| <!DOCTYPE html> | ||
| <html lang="zh-CN"> | ||
|
|
||
| <head> | ||
| <meta charset="UTF-8"> | ||
| <meta name="viewport" | ||
| content="width=device-width, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0"> | ||
| <meta http-equiv="X-UA-Compatible" content="ie=edge"> | ||
|
|
||
| <meta name="author" content="修年"> | ||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
| <title>Indexing | 修年</title> | ||
|
|
||
|
|
||
|
|
||
| <link rel="icon" href="/favicon.ico"> | ||
|
|
||
|
|
||
|
|
||
|
|
||
| <!-- stylesheets list from _config.yml --> | ||
|
|
||
| <link rel="stylesheet" href="/css/style.css"> | ||
|
|
||
|
|
||
|
|
||
|
|
||
| <!-- scripts list from _config.yml --> | ||
|
|
||
| <script src="/js/script.js"></script> | ||
|
|
||
| <script src="/js/tocbot.min.js"></script> | ||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
| <meta charset="UTF-8"> | ||
| <title>live2d-demo</title> | ||
| <script src="https://apps.bdimg.com/libs/jquery/2.1.4/jquery.min.js"></script> | ||
| <!-- Live2DCubismCore --> | ||
| <script src="https://cdn.jsdelivr.net/gh/litstronger/live2d-moc3@master/js/frame/live2dcubismcore.min.js"></script> | ||
| <!-- Include Pixi. --> | ||
| <script src="https://cdnjs.cloudflare.com/ajax/libs/pixi.js/4.6.1/pixi.min.js"></script> | ||
| <!-- Include Cubism Components. --> | ||
| <script src="https://cdn.jsdelivr.net/gh/litstronger/live2d-moc3@master/js/live2dcubismframework.js"></script> | ||
| <script src="https://cdn.jsdelivr.net/gh/litstronger/live2d-moc3@master/js/live2dcubismpixi.js"></script> | ||
| <!-- User's Script --> | ||
| <script src="https://cdn.jsdelivr.net/gh/litstronger/live2d-moc3@master/js/l2d.js"></script> | ||
| <script src="https://cdn.jsdelivr.net/gh/litstronger/live2d-moc3@master/js/main.js"></script> | ||
| <style> | ||
| </style> | ||
| <meta name="generator" content="Hexo 5.4.2"></head> | ||
|
|
||
| <body> | ||
| <script> | ||
| // this function is used to check current theme before page loaded. | ||
| (() => { | ||
| const currentTheme = window.localStorage && window.localStorage.getItem('theme') || ''; | ||
| const isDark = currentTheme === 'dark'; | ||
| const pagebody = document.getElementsByTagName('body')[0] | ||
| if (isDark) { | ||
| pagebody.classList.add('dark-theme'); | ||
| // mobile | ||
| document.getElementById("mobile-toggle-theme").innerText = "· Dark" | ||
| } else { | ||
| pagebody.classList.remove('dark-theme'); | ||
| // mobile | ||
| document.getElementById("mobile-toggle-theme").innerText = "· Light" | ||
| } | ||
| })(); | ||
| </script> | ||
|
|
||
| <div class="wrapper"> | ||
| <header> | ||
| <nav class="navbar"> | ||
| <div class="container"> | ||
| <div class="navbar-header header-logo"><a href="/">Xiunian's Blog</a></div> | ||
| <div class="menu navbar-right"> | ||
|
|
||
| <a class="menu-item" href="/about">About</a> | ||
|
|
||
| <a class="menu-item" href="/archives">Posts</a> | ||
|
|
||
| <a class="menu-item" href="/tag">Tags</a> | ||
|
|
||
| <input id="switch_default" type="checkbox" class="switch_default"> | ||
| <label for="switch_default" class="toggleBtn"></label> | ||
| </div> | ||
| </div> | ||
| </nav> | ||
|
|
||
|
|
||
| <nav class="navbar-mobile" id="nav-mobile"> | ||
| <div class="container"> | ||
| <div class="navbar-header"> | ||
| <div> | ||
| <a href="/">Xiunian's Blog</a><a id="mobile-toggle-theme">· Light</a> | ||
| </div> | ||
| <div class="menu-toggle" onclick="mobileBtn()">☰ Menu</div> | ||
| </div> | ||
| <div class="menu" id="mobile-menu"> | ||
|
|
||
| <a class="menu-item" href="/about">About</a> | ||
|
|
||
| <a class="menu-item" href="/archives">Posts</a> | ||
|
|
||
| <a class="menu-item" href="/tag">Tags</a> | ||
|
|
||
| </div> | ||
| </div> | ||
| </nav> | ||
|
|
||
| </header> | ||
| <script> | ||
| var mobileBtn = function f() { | ||
| var toggleMenu = document.getElementsByClassName("menu-toggle")[0]; | ||
| var mobileMenu = document.getElementById("mobile-menu"); | ||
| if(toggleMenu.classList.contains("active")){ | ||
| toggleMenu.classList.remove("active") | ||
| mobileMenu.classList.remove("active") | ||
| }else{ | ||
| toggleMenu.classList.add("active") | ||
| mobileMenu.classList.add("active") | ||
| } | ||
| } | ||
| </script> | ||
| <div class="main"> | ||
| <div class="container"> | ||
|
|
||
|
|
||
| <div class="post-toc" style="right: -4em;"> | ||
| <div class="tocbot-list"> | ||
| </div> | ||
| <div class="tocbot-list-menu"> | ||
| <a class="tocbot-toc-expand" onclick="expand_toc()">Expand all</a> | ||
| <a onclick="go_top()">Back to top</a> | ||
| <a onclick="go_bottom()">Go to bottom</a> | ||
| </div> | ||
| </div> | ||
|
|
||
| <script> | ||
| var tocbot_timer; | ||
| var DEPTH_MAX = 6; // 为 6 时展开所有 | ||
| var tocbot_default_config = { | ||
| tocSelector: '.tocbot-list', | ||
| contentSelector: '.post-content', | ||
| headingSelector: 'h1, h2, h3, h4, h5', | ||
| orderedList: false, | ||
| scrollSmooth: true, | ||
| onClick: extend_click, | ||
| }; | ||
|
|
||
| function extend_click() { | ||
| clearTimeout(tocbot_timer); | ||
| tocbot_timer = setTimeout(function() { | ||
| tocbot.refresh(obj_merge(tocbot_default_config, { | ||
| hasInnerContainers: true | ||
| })); | ||
| }, 420); // 这个值是由 tocbot 源码里定义的 scrollSmoothDuration 得来的 | ||
| } | ||
|
|
||
| document.ready(function() { | ||
| tocbot.init(obj_merge(tocbot_default_config, { | ||
| collapseDepth: 1 | ||
| })); | ||
| }); | ||
|
|
||
| function expand_toc() { | ||
| var b = document.querySelector('.tocbot-toc-expand'); | ||
| var expanded = b.getAttribute('data-expanded'); | ||
| expanded ? b.removeAttribute('data-expanded') : b.setAttribute('data-expanded', true); | ||
| tocbot.refresh(obj_merge(tocbot_default_config, { | ||
| collapseDepth: expanded ? 1 : DEPTH_MAX | ||
| })); | ||
| b.innerText = expanded ? 'Expand all' : 'Collapse all'; | ||
| } | ||
|
|
||
| function go_top() { | ||
| window.scrollTo(0, 0); | ||
| } | ||
|
|
||
| function go_bottom() { | ||
| window.scrollTo(0, document.body.scrollHeight); | ||
| } | ||
|
|
||
| function obj_merge(target, source) { | ||
| for (var item in source) { | ||
| if (source.hasOwnProperty(item)) { | ||
| target[item] = source[item]; | ||
| } | ||
| } | ||
| return target; | ||
| } | ||
| </script> | ||
|
|
||
|
|
||
|
|
||
| <article class="post-wrap"> | ||
| <header class="post-header"> | ||
| <h1 class="post-title">Indexing</h1> | ||
|
|
||
| <div class="post-meta"> | ||
|
|
||
| Author: <a itemprop="author" rel="author" href="/">修年</a> | ||
|
|
||
|
|
||
|
|
||
| <span class="post-time"> | ||
| Date: <a href="#">十一月 11, 2023 14:25:29</a> | ||
| </span> | ||
|
|
||
|
|
||
| </div> | ||
|
|
||
| </header> | ||
|
|
||
| <div class="post-content"> | ||
| <h2 id="sparce-indexing"><a href="#sparce-indexing" class="headerlink" title="sparce indexing"></a>sparce indexing</h2><p>基于chunks的去重都要求使用full index，而这RAM一般承受不起，但是纯用disk io就太慢了。所以它利用了数据局部性：</p> | ||
| <p>If two pieces of backup streams share any chunks, they are likely to share many chunks. <u>如果两个segment共享了某个chunk，那么它们很有可能共享很多chunks。</u></p> | ||
| <p><strong>是这样的流程：</strong></p> | ||
| <ol> | ||
| <li>分段为segment；</li> | ||
| <li>计算该segment的每个chunk的fp，然后对每个chunk查询其对应的sparce indexing table: <fp, segment_id>，记录<u>可能跟它共享很多chunk的segment</u>的segment_id；</li> | ||
| <li>读取这些segment_id对应的segment的chunk indexing table（存储在disk中）；</li> | ||
| <li>for every chunks： 重复，copy entry ；不重复，add to new container</li> | ||
| <li>最后再将该segment的信息写入磁盘，填写sparce indexing表。</li> | ||
| </ol> | ||
| <p>而sparce indexing表最一开始，由对input segment进行chunks的随机抽样得出（或者逐渐构建起来，反正大概是这个意思）</p> | ||
| <p>可以看到，它将segment info保留在disk中，在RAM中只保留fp2seg_id的映射，每次只需简单从磁盘中读取几个segment info即可，利用数据局部性极大地降低了磁盘IO次数。</p> | ||
| <p>Odess采用的就是类似这种capping+sparce indexing的方法。</p> | ||
| <p>将sparce indexing从原来的<fp, seg_id>改为<fp, cid>，并且每次只取top T个包含sample chunks最多的容器，从而将对segment进行cap修改为对container进行cap。仔细想想，这样确实依然保证了原算法的核心思想，也属于是segment size = container size的特种了。</p> | ||
| <p>不这个<fp, cid>不就是Odess中的recipe（或者说是全局指纹表）吗？乐。Odess也确实体现了这种capping+sparce indexing结合的方法【只不过进行简化了，每个chunk固定取其第一个container】。</p> | ||
|
|
||
| </div> | ||
|
|
||
|
|
||
| <section class="post-tags"> | ||
| <div> | ||
| <span>Tag(s):</span> | ||
| <span class="tag"> | ||
|
|
||
| </span> | ||
| </div> | ||
| <div> | ||
| <a href="javascript:window.history.back();">back</a> | ||
| <span>· </span> | ||
| <a href="/">home</a> | ||
| </div> | ||
| </section> | ||
| <section class="post-nav"> | ||
|
|
||
| <a class="prev" rel="prev" href="/2023/11/18/compilation_principle/">编译原理</a> | ||
|
|
||
|
|
||
| <a class="next" rel="next" href="/2023/10/27/driver_develop/">驱动开发小记</a> | ||
|
|
||
| </section> | ||
|
|
||
|
|
||
| </article> | ||
| </div> | ||
|
|
||
| </div> | ||
| <footer id="footer" class="footer"> | ||
| <div class="copyright"> | ||
| <span>© 修年 | Powered by <a href="https://hexo.io" target="_blank">Hexo</a> & <a href="https://github.com/Siricee/hexo-theme-Chic" target="_blank">Chic</a></span> | ||
| </div> | ||
| </footer> | ||
|
|
||
| </div> | ||
| </body> | ||
|
|
||
| </html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes
File renamed without changes
File renamed without changes
File renamed without changes
File renamed without changes
File renamed without changes
File renamed without changes
Oops, something went wrong.