feat: express 爬虫流程

GuoYuFu123 · GuoYuFu123 · commit e4bc177cb84a · 2020-05-26T14:07:50.000+08:00
diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,8 @@
 /typescript进阶/dist
 /typescript进阶/package-lock.json
 /typescript进阶/.parcel-cache
+
+#crowller-code-express ignore
+/crowller-code-express/node_modules
+/crowller-code-express/build
+/crowller-code-express/package-lock.json
diff --git a/crowller-code-express/data/course.json b/crowller-code-express/data/course.json
@@ -0,0 +1 @@
+{"1590472484271":[{"index":0,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":1,"title":"React 16.4 开发简书项目 从零基础入门到实战"},{"index":2,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":3,"title":"从基础到实战\t手把手带你掌握新版Webpack4.0"},{"index":4,"title":"Dell Lee 的微课堂，职业规划答疑解惑，精彩文章与你分享"}],"1590472490515":[{"index":0,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":1,"title":"React 16.4 开发简书项目 从零基础入门到实战"},{"index":2,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":3,"title":"从基础到实战\t手把手带你掌握新版Webpack4.0"},{"index":4,"title":"Dell Lee 的微课堂，职业规划答疑解惑，精彩文章与你分享"}],"1590473203446":[{"index":0,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":1,"title":"React 16.4 开发简书项目 从零基础入门到实战"},{"index":2,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":3,"title":"从基础到实战\t手把手带你掌握新版Webpack4.0"},{"index":4,"title":"Dell Lee 的微课堂，职业规划答疑解惑，精彩文章与你分享"}]}
diff --git a/crowller-code-express/package.json b/crowller-code-express/package.json
@@ -0,0 +1,37 @@
+{
+  "name": "crowller-code",
+  "version": "1.0.0",
+  "description": "typescript crowller",
+  "main": "index.js",
+  "scripts": {
+    "dev:ts": "ts-node ./src/crowller.ts",
+    "dev:build": "tsc -w",
+    "dev:start": "nodemon node ./build/index.js",
+    "dev": "tsc && concurrently \"npm run dev:build\" \"npm run dev:start\""
+  },
+  "nodemonConfig": {
+    "ignore": [
+      "data/*"
+    ]
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/cheerio": "^0.22.18",
+    "@types/cookie-session": "^2.0.39",
+    "@types/express": "^4.17.6",
+    "@types/superagent": "^4.1.7",
+    "concurrently": "^5.2.0",
+    "nodemon": "^2.0.3",
+    "ts-node": "^8.9.1",
+    "typescript": "^3.8.3"
+  },
+  "dependencies": {
+    "body-parser": "^1.19.0",
+    "cheerio": "^1.0.0-rc.3",
+    "cookie-session": "^1.4.0",
+    "express": "^4.17.1",
+    "superagent": "^5.2.2"
+  }
+}
diff --git a/crowller-code-express/readme.md b/crowller-code-express/readme.md
@@ -0,0 +1,32 @@
+# typescript的node爬虫
+
+###### author： 果果
+
+#### 构建
+
+```
+npm i or yarn
+```
+
+#### 运行
+
+```
+#运行ts
+npm run dev:ts
+
+#将ts转为js
+npm run dev:build
+
+#运行打包之后js
+npm run dev:start
+
+#将ts打包并自动编译[运行这个即可]
+npm run dev
+```
+
+#### 结语
+
+```
+技术交流，共同进步!
+```
+
diff --git a/crowller-code-express/src/crowller.ts b/crowller-code-express/src/crowller.ts
@@ -0,0 +1,39 @@
+// ts -> .d.ts -> js 
+// 如果我们的项目是ts文件，但是引入是的js的文件，这样就需要一个.d.ts的翻译文件 @types/superagent
+import path from 'path';
+import fs from 'fs';
+import superagent from 'superagent'; // 爬虫的包
+
+
+export interface Analyzer {
+  analyze: (html: string, filePath: string) => string
+}
+
+class Crowller {
+  private readonly filePath = path.resolve(__dirname, '../data/course.json');
+
+  constructor(private url: string, private anlayzer: Analyzer) {
+    this.initSpiderProcess()
+  }
+  // 爬虫入口
+  async initSpiderProcess() {
+    const html = await this.getRawHtml();
+    const fileContent = this.anlayzer.analyze(html, this.filePath);
+    fileContent && this.writeFile(fileContent)
+  }
+  // 获取原始html
+  async getRawHtml() {
+    const result = await superagent.get(this.url);
+    return result.text;
+  }
+  // 写文件
+  writeFile(content: string) {
+    fs.writeFileSync(this.filePath, content);
+  }
+}
+
+export default Crowller;
+
+
+
+
diff --git a/crowller-code-express/src/d.ts/custom.d.ts b/crowller-code-express/src/d.ts/custom.d.ts
@@ -0,0 +1,5 @@
+declare namespace Express {
+  interface Request {
+    teacherName: string
+  }
+}
diff --git a/crowller-code-express/src/dellAnalyzer.ts b/crowller-code-express/src/dellAnalyzer.ts
@@ -0,0 +1,60 @@
+import fs from 'fs';
+import cheerio from 'cheerio'; // 数据截取的包
+import { Analyzer } from './crowller';
+
+interface Course {
+  index: number;
+  title: string;
+}
+interface CourseResult {
+  time: number,
+  data: Course[]
+}
+interface Content {
+  [propName: number]: Course[];
+}
+export default class DellAnalyzer implements Analyzer {
+  private static instance: DellAnalyzer
+  static getInstance() {
+    if (!DellAnalyzer.instance) {
+      DellAnalyzer.instance = new DellAnalyzer();
+    }
+    return DellAnalyzer.instance;
+  } 
+  private constructor () {
+     
+  }
+   // 获取课程信息
+   private getCourseInfo(html: string) {
+    const $ = cheerio.load(html);
+    const courseItems = $('.course-item');
+    const courseInfos: Course[] = [];
+    courseItems.map((index, element) => {
+      const desc = $(element).find(".course-desc");
+      const title = desc.text();
+      courseInfos.push({ index, title })
+    })
+    const result = {
+      time: new Date().getTime(),
+      data: courseInfos
+    }
+    return result;
+  }
+  // 存储到json文件[先读取文件，如果有就添加。没有就创建]
+  private generateJsonContent(courseInfos: CourseResult, filePath:string) {
+    let fileContent: Content = {};
+    if (fs.existsSync(filePath)) {
+      const readFileContent = fs.readFileSync(filePath, 'utf-8');
+      readFileContent && (fileContent = JSON.parse(readFileContent));
+    }
+    fileContent[courseInfos.time] = courseInfos.data;
+    return fileContent;
+    
+  }
+  // 提供外部调用的分析接口
+  public analyze(html: string, filePath:string) {
+    const courseInfos = this.getCourseInfo(html);
+    const fileContent =  this.generateJsonContent(courseInfos, filePath);
+    return JSON.stringify(fileContent);
+  }
+}
diff --git a/crowller-code-express/src/index.ts b/crowller-code-express/src/index.ts
@@ -0,0 +1,28 @@
+import express, {Request, Response, NextFunction} from "express";
+import bodyParser from 'body-parser';
+import cookieSession from 'cookie-session';
+import router from './router';
+
+//1。express 库的类型定义文件 .d.ts的文件类型描述不准确
+//2。 当使用中间件之后，对req和res 做个修改，实际上类型没有发生改变
+
+const app = express();
+// 中间件使用
+app.use(bodyParser.urlencoded({ extended: false }))
+app.use(cookieSession({
+  name: 'session',
+  keys: ['teacher!lkjsldj'],
+  // Cookie Options
+  maxAge: 24 * 60 * 60 * 1000 // 24 hours
+}))
+// 中间件问题, 其实中间件就是一个函数
+// app.use((req: Request, res: Response, next: NextFunction ) => {
+//   req.teacherName = 'guoguo';
+//   next();
+// })
+
+app.use(router)
+
+app.listen(7001, () => {
+  console.log("server is running at 7001");
+})
diff --git a/crowller-code-express/src/router.ts b/crowller-code-express/src/router.ts
@@ -0,0 +1,101 @@
+import { Router, Request, Response } from 'express';
+import Crowller from './crowller';
+import DellAnalyzer from './dellAnalyzer';
+import fs from 'fs';
+import path from 'path';
+
+const router = Router();
+
+interface RequestWithBody extends Request {
+  body: {
+    [key: string]: string | undefined;
+  }
+}
+
+router.get('/', (req: Request, res: Response) => {
+  const isLogin = req.session ? req.session.login : undefined;
+  if (isLogin) {
+    res.send(`
+    <html>
+    <body>
+      <a href="/getData">爬取数据</a>
+      <a href="/showData">展示数据</a>
+      <a href="/logout">退出</a>
+    </body>
+  </html> 
+    `)
+  } else {
+    res.send(`
+      <html>
+        <body>
+          <form method="post" action="/login">
+            <input type="password" name="password"></input>
+            <button>提交</button>
+          </form>
+        </body>
+      </html> 
+      `)
+  }
+
+})
+
+router.post('/login', (req: RequestWithBody, res: Response) => {
+  const { password } = req.body;
+  const isLogin = req.session ? req.session.login : undefined;
+  if (isLogin) {
+    res.send('已经登录过')
+  } else {
+    if (password === '123') {
+      if (req.session) {
+        req.session.login = true;
+        setTimeout(() => {
+          res.redirect('/');
+        }, 2000)
+      }
+    } else {
+      res.send('登录失败')
+    }
+  }
+})
+
+router.get('/logout', (req: RequestWithBody, res: Response) => {
+  if (req.session) {
+    req.session.login = undefined;
+  }
+  res.redirect('/');
+  // res.send("已经退出了")
+})
+
+router.get('/getData', (req: RequestWithBody, res: Response) => {
+  const isLogin = req.session ? req.session.login : false;
+  if (isLogin) {
+    // 爬虫url：http://www.dell-lee.com/typescript/demo.html?secret=secret
+    const secret = 'secretKey';
+    const url = `http://www.dell-lee.com/typescript/demo.html?secret=${secret}`;
+    const anlayzer = DellAnalyzer.getInstance();
+    new Crowller(url, anlayzer);
+    res.send('getData success')
+  } else {
+    res.send('请登录后进行爬取')
+  }
+})
+
+router.get('/showData', (req: RequestWithBody, res: Response) => {
+  const isLogin = req.session ? req.session.login : false;
+  if (isLogin) {
+    try {
+      const position = path.resolve(__dirname, '../data/course.json');
+      const result = fs.readFileSync(position, 'utf-8');
+      res.json(JSON.parse(result));
+    } catch (e) {
+      res.send("尚未爬取到数据")
+    }
+  } else {
+    res.send("用户尚未登录，请登录之后进行数据展示")
+  }
+})
+
+router.get('/*', (req: Request, res: Response) => {
+  res.send('404');
+})
+export default router;
diff --git a/crowller-code-express/tsconfig.json b/crowller-code-express/tsconfig.json
@@ -0,0 +1,66 @@
+{
+  "compilerOptions": {
+    /* Basic Options */
+    // "incremental": true,                   /* Enable incremental compilation */
+    "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    // "lib": [],                             /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                       /* Allow javascript files to be compiled. */
+    // "checkJs": true,                       /* Report errors in .js files. */
+    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    // "sourceMap": true,                     /* Generates corresponding '.map' file. */
+    // "outFile": "./",                       /* Concatenate and emit output to single file. */
+    "outDir": "./build",                        /* Redirect output structure to the directory. */
+    "rootDir": "./src",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                     /* Enable project compilation */
+    // "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+    // "removeComments": true,                /* Do not emit comments to output. */
+    // "noEmit": true,                        /* Do not emit outputs. */
+    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true,                           /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,              /* Enable strict null checks. */
+    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    // "noUnusedLocals": true,                /* Report errors on unused locals. */
+    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+    /* Module Resolution Options */
+    // "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [],                       /* List of folders to include type definitions from. */
+    // "types": [],                           /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true,                  /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+    // "allowUmdGlobalAccess": true,          /* Allow accessing UMD globals from modules. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+
+    /* Advanced Options */
+    "forceConsistentCasingInFileNames": true  /* Disallow inconsistently-cased references to the same file. */
+  }
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+{"1590472484271":[{"index":0,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":1,"title":"React 16.4 开发简书项目从零基础入门到实战"},{"index":2,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":3,"title":"从基础到实战\t手把手带你掌握新版Webpack4.0"},{"index":4,"title":"Dell Lee 的微课堂，职业规划答疑解惑，精彩文章与你分享"}],"1590472490515":[{"index":0,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":1,"title":"React 16.4 开发简书项目从零基础入门到实战"},{"index":2,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":3,"title":"从基础到实战\t手把手带你掌握新版Webpack4.0"},{"index":4,"title":"Dell Lee 的微课堂，职业规划答疑解惑，精彩文章与你分享"}],"1590473203446":[{"index":0,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":1,"title":"React 16.4 开发简书项目从零基础入门到实战"},{"index":2,"title":"Vue2.5开发去哪儿网App 从零基础入门到实战项目"},{"index":3,"title":"从基础到实战\t手把手带你掌握新版Webpack4.0"},{"index":4,"title":"Dell Lee 的微课堂，职业规划答疑解惑，精彩文章与你分享"}]}