Merge pull request #250 from osamhack2021/develop

Develop

AI(BE)/riskout/extractive/README.md

@@ -0,0 +1,10 @@
+Extractive Summarization
+========================
+
+## Gensim summarizer?
+- Summarizing is based on ranks of text sentences using a variation of the TextRank algorithm.
+
+
+## References
+- [RaRe-Technologies/gensim](https://github.com/RaRe-Technologies/gensim)
+  - [Variations of the Similarity Function of TextRank for Automated Summarization, 2016](https://arxiv.org/abs/1602.03606)

AI(BE)/riskout/fakenews/README.md

@@ -0,0 +1,19 @@
+Fake News Classifier
+========================
+## Model
+| Model Accuracy | Train | Validation | Test |
+|----------------|-------|------------|------|
+| LSTM           | 0.9774 | 0.98381   | 0.7544 |
+| SenCNN         | 0.9624 | 0.8016    | 0.7827 |
+| BERT           | 0.9662 | 0.8299    | 0.8070 |
+
+- 세 개 모델 모두 Early Stopping 을 적용한 결과입니다.
+
+## Dataset
+데이터셋은 [SNU factcheck](https://factcheck.snu.ac.kr/)를 크롤링하여 사실, 거짓으로 라벨링하였음.
+거짓 데이터셋의 양이 더 많아서 네이버 뉴스에서 추가로 데이터를 모아 두 라벨링된 데이터의 크기가 같게 함.
+
+## References
+- [A Study on Korean Fake news Detection Model Using Word Embedding, 2020](https://www.koreascience.or.kr/article/CFKO202022449680088.pdf)
+- [Research Analysis in Automatic Fake News Detection, 2019](http://hiai.co.kr/wp-content/uploads/2019/12/%EB%85%BC%EB%AC%B8%EC%A6%9D%EB%B9%99_2019_02.pdf)
+

WEB(BE)/analyzer/analyzer.py

@@ -312,7 +312,7 @@ def main():
         if tup[9] not in date_list:
             date_list.append(tup[9])
 
-    for date in date_list:
+    for date in date_list[:len(date_list) - 1]:
         cur.execute("SELECT * FROM CrawlContents WHERE isAnalyzed = 0 AND category = 'news' AND created_at = ?", (date,))
         ranked_list = dataRanker(cur.fetchall())
 
@@ -321,7 +321,8 @@ def main():
             cur.execute("UPDATE CrawlContents SET isAnalyzed = 1 WHERE isAnalyzed = 0 AND category = 'news' AND created_at = ?", (date,))
             conn.commit()
 
-    cur.execute("SELECT * FROM CrawlContents WHERE isAnalyzed = 0") # news는 이미 analyzed 되었기 때문에 sns와 community만 남는다
+    # 오늘 이전의 news는 이미 analyzed 되었기 때문에 "크롤링 시점 오늘 뉴스 전부", sns, community가 남는다
+    cur.execute("SELECT * FROM CrawlContents WHERE isAnalyzed = 0") 
     important_data_list.extend(cur.fetchall())
 
 

WEB(BE)/crawler/crawler/model/naver/NaverNewsSite.py

@@ -75,7 +75,7 @@ def get_articleID(self, contents_url):
         return article_id
 
     def contentCheck(self, content):
-        if(len(content.body) < 800 or len(content.body) > 2000):
+        if(len(content.body) < 500 or len(content.body) > 2000):
             raise length_error
 
         return

WEB(BE)/drf/detection_status/views.py

@@ -831,7 +831,7 @@ def getAnalyzedData(self, articleIds, period):
         db_result = mongo.find_item(query, "riskout", "analyzed")
         db_filtered = self.datetimeFormatter([v for _, v in enumerate(db_result)]) if (db_result.count()) else []
 
-        now = datetime.utcnow() + timedelta(hours=9)
+        now = datetime.utcnow() + timedelta(hours=9) - timedelta(days=4)
         today_datetime = datetime(now.year, now.month, now.day)
         today = datetime(now.year, now.month, now.day).strftime('%y-%m-%d')
 

WEB(BE)/drf/secrets.example.json

@@ -1,11 +1,11 @@
 {
-    "SECRET_KEY": "asdfasdfasdfasdf",
-    "DEBUG": "True",
-    "ALLOWED_HOSTS": ["localhost", "http://localhost:3000"],
-    "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend",
-    "EMAIL_HOST": "smtp.googlemail.com",
-    "EMAIL_USE_TLS": "True",
-    "EMAIL_PORT": 587,
-    "EMAIL_HOST_USER": "[email protected]",
-    "EMAIL_HOST_PASSWORD": "asdfasdfasdf*"
+  "SECRET_KEY": "asdfasdfasdfasdf",
+  "DEBUG": "False",
+  "ALLOWED_HOSTS": ["localhost", "http://localhost:3000"],
+  "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend",
+  "EMAIL_HOST": "smtp.googlemail.com",
+  "EMAIL_USE_TLS": "True",
+  "EMAIL_PORT": 587,
+  "EMAIL_HOST_USER": "[email protected]",
+  "EMAIL_HOST_PASSWORD": "asdfasdfasdf*"
 }

WEB(FE)/frontend/public/static/ReportData.example.json → WEB(FE)/frontend/public/static/data/ReportData.example.json

@@ -1,6 +1,7 @@
 {
   "overview": "북한 대외선전매체 우리민족끼리 와 통일의 메아리, 조선의 오늘 은 각각 김정은 집권 10년을 기념하는 특별 웹페이지를 신설하고 홈페이지 첫 화면에 연결 배너를 띄웠다.",
   "period": 24,
+  "isDone": true,
   "briefingGraphData": {
     "secretsCount": 52,
     "fakeNewsCount": 4,

WEB(FE)/frontend/src/components/DetectionStatus/DetectionTable.js

@@ -14,8 +14,8 @@ import {
   Typography,
 } from '@mui/material';
 import SecretsTableRow from './SecretsTableRow';
+import { searchState, useContents } from '../../atoms/searchState';
 import { useSessionStorage } from '../../js/util';
-import { useContents } from '../../atoms/searchState';
 import useSearchInitEffect from '../../hooks/useSearchInitEffect';
 
 import { useTheme } from '@mui/material/styles';
@@ -24,6 +24,7 @@ import KeyboardArrowLeft from '@mui/icons-material/KeyboardArrowLeft';
 import KeyboardArrowRight from '@mui/icons-material/KeyboardArrowRight';
 import LastPageIcon from '@mui/icons-material/LastPage';
 import PropTypes from 'prop-types';
+import { useRecoilValue } from 'recoil';
 
 function TablePaginationActions(props) {
   const theme = useTheme();
@@ -97,9 +98,9 @@ TablePaginationActions.propTypes = {
 export default function DetectionTable({ showDetailModal, toggleScrap }) {
   useSearchInitEffect();
   const contents = useContents();
+  const { isDone } = useRecoilValue(searchState);
   const [page, setPage] = useState(0);
   const [rowsPerPage, setRowsPerPage] = useState(4);
-
   // Avoid a layout jump when reaching the last page with empty rows.
   const emptyRows =
     page > 0 ? Math.max(0, (1 + page) * rowsPerPage - contents.length) : 0;
@@ -165,41 +166,53 @@ export default function DetectionTable({ showDetailModal, toggleScrap }) {
             </TableCell>
           </TableRow>
         </TableHead>
-        <TableBody>
-          {contents &&
-            (rowsPerPage > 0
-              ? contents.slice(
-                  page * rowsPerPage,
-                  page * rowsPerPage + rowsPerPage
-                )
-              : contents
-            ).map((article, id) => (
-              <SecretsTableRow
-                key={id}
-                isAlreadyScrapped={isInCart(article._id)}
-                {...{ article, showDetailModal, toggleScrap }}
-              />
-            ))}
-          {emptyRows > 0 && (
-            <TableRow style={{ height: 53 * emptyRows }}>
-              <TableCell colSpan={6} />
-            </TableRow>
-          )}
-        </TableBody>
-        <TableFooter>
-          <TableRow>
-            <TablePagination
-              rowsPerPageOptions={[4]}
-              colSpan={4}
-              count={contents.length}
-              rowsPerPage={rowsPerPage}
-              page={page}
-              onPageChange={handleChangePage}
-              onRowsPerPageChange={handleChangeRowsPerPage}
-              ActionsComponent={TablePaginationActions}
-            />
-          </TableRow>
-        </TableFooter>
+        {contents || isDone ? (
+          <>
+            <TableBody>
+              {(rowsPerPage > 0
+                ? contents.slice(
+                    page * rowsPerPage,
+                    page * rowsPerPage + rowsPerPage
+                  )
+                : contents
+              ).map((article, id) => (
+                <SecretsTableRow
+                  key={id}
+                  isAlreadyScrapped={isInCart(article._id)}
+                  {...{ article, showDetailModal, toggleScrap }}
+                />
+              ))}
+              {emptyRows > 0 && (
+                <TableRow style={{ height: 53 * emptyRows }}>
+                  <TableCell colSpan={6} />
+                </TableRow>
+              )}
+            </TableBody>
+            <TableFooter>
+              <TableRow>
+                <TablePagination
+                  rowsPerPageOptions={[4]}
+                  colSpan={4}
+                  count={contents.length}
+                  rowsPerPage={rowsPerPage}
+                  page={page}
+                  onPageChange={handleChangePage}
+                  onRowsPerPageChange={handleChangeRowsPerPage}
+                  ActionsComponent={TablePaginationActions}
+                />
+              </TableRow>
+            </TableFooter>
+          </>
+        ) : (
+          <TableBody>
+            <TableCell></TableCell>
+            <TableCell align="center">
+              <Typography>현재 데이터가 존재하지 않습니다.</Typography>
+            </TableCell>
+            <TableCell></TableCell>
+            <TableCell></TableCell>
+          </TableBody>
+        )}
       </Table>
     </TableContainer>
   );

WEB(FE)/frontend/src/components/Modal/LoginModal.js

@@ -34,6 +34,30 @@ const LoginModal = (props) => {
     setUserPassword(e.target.value);
   };
 
+  const onSubmit = (e) => {
+    e.preventDefault();
+    fetch('/api/user/login/', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(data),
+    })
+      .then((res) => res.json())
+      .then((json) => {
+        if (json.token) {
+          props.userHasAuthenticated(true, data.username, json.token);
+          alert('환영합니다.' + username + '님.');
+          history.push('/presstrends');
+          props.setModal(true);
+          console.log(json);
+        } else {
+          alert('아이디 또는 비밀번호를 확인해주세요.');
+        }
+      })
+      .catch((error) => alert(error));
+  };
+
   const paperStyle = {
     padding: '60px 68px 40px',
     width: 450,
@@ -52,7 +76,7 @@ const LoginModal = (props) => {
   };
 
   const onClick = () => {
-    alert('데모 버전에서는 제공하지 않는 기능입니다')
+    alert('베타 버전은 사전에 승인된 인원만 사용 가능합니다.')
   };
 
   const fetchLoginApi = (e) => {
@@ -93,43 +117,50 @@ const LoginModal = (props) => {
         <Grid align="left">
           <h1 style={{ fontSize: '32px' }}>로그인</h1>
         </Grid>
-        <Grid align="center">
-          <Box
-            sx={{ width: 314, height: 50, marginBottom: '1.2em', marginTop: 5 }}
-          >
-            <TextField
-              label="아이디"
-              placeholder="아이디를 입력해 주세요"
-              fullWidth
-              required
-              onChange={handleNameChange}
-              variant="outlined"
-            />
-          </Box>
-          <Box sx={{ width: 314, height: 50, marginBottom: '2em' }}>
-            <TextField
-              label="비밀번호"
-              placeholder="비밀번호를 입력해 주세요."
-              fullWidth
-              type="password"
-              required
-              variant="outlined"
-              onChange={handlePasswordChange}
-              onKeyDown={keyDown}
-            />
-          </Box>
-          {/* */}
-          <Link sx={{ textDecoration: 'none' }}>
-            <Button
-              variant="contained"
-              style={btnstyle}
-              className="JoinLoign-button"
-              onClick={fetchLoginApi}
+
+        <form onSubmit={onSubmit}>
+          <Grid align="center">
+            <Box
+              sx={{
+                width: 314,
+                height: 50,
+                marginBottom: '1.2em',
+                marginTop: 5,
+              }}
             >
-              로그인
-            </Button>
-          </Link>
-        </Grid>
+              <TextField
+                label="아이디"
+                placeholder="아이디를 입력해 주세요"
+                fullWidth
+                required
+                onChange={handleNameChange}
+                variant="outlined"
+              />
+            </Box>
+            <Box sx={{ width: 314, height: 50, marginBottom: '2em' }}>
+              <TextField
+                label="비밀번호"
+                placeholder="비밀번호를 입력해 주세요."
+                fullWidth
+                type="password"
+                required
+                variant="outlined"
+                onChange={handlePasswordChange}
+              />
+            </Box>
+            {/* */}
+            <Link sx={{ textDecoration: 'none' }}>
+              <Button
+                type="submit"
+                variant="contained"
+                style={btnstyle}
+                className="JoinLoign-button"
+              >
+                로그인
+              </Button>
+            </Link>
+          </Grid>
+        </form>
         <Box sx={{ display: 'flex', marginBottom: '5em' }}>
           <Typography variant="subtitle1" sx={{}}>
             <input type="checkbox" style={{ marginTop: '7px' }} />

WEB(FE)/frontend/src/hooks/useSearch.js

@@ -27,29 +27,25 @@ export default function useSeacrh() {
       alert('로그인이 필요한 화면 입니다.');
       history.push('/login');
     } else {
-      fetch('/api/nlp/analyze/', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: `Token ${token}`,
-        },
-        body: JSON.stringify(data),
-      })
-        .then((res) => res.json())
-        .then((json) => {
-          console.log('=========[api/nlp/analyze/]========');
-          console.log(json);
+      if (process.env.REACT_APP_USE_STATIC_RESPONSE == 'True') {
+        const searchUrl = `/static/SecretData.example.json`;
+        client.get(searchUrl).then((data) => {
+          setSearchList(data.data);
         });
+      } else {
+        fetch('/api/nlp/analyze/', {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Token ${token}`,
+          },
+          body: JSON.stringify(data),
+        })
+          .then((res) => res.json())
+          .then((json) => {
+            console.log(json);
+          });
+      }
     }
-
-    const searchUrl = `/static/SecretData.example.json`;
-    async function fetchSearch() {
-      client.get(searchUrl).then((data) => {
-        setSearchList(data.data);
-        console.log(data.data);
-      });
-    }
-
-    fetchSearch();
   }, [filterList]);
 }

WEB(FE)/frontend/src/hooks/useSearchInitEffect.js

@@ -10,7 +10,7 @@ export default function useSearchInitEffect() {
   const searchSetting = useRecoilValue(searchSettingState);
   /* TODO searchSetting 을 이용해서 params 넘겨주는 코드 작성 */
   useEffect(() => {
-    if (Object.keys(search.contents).length !== 0) return;
+    if (!search.contents || Object.keys(search.contents).length !== 0) return;
 
     //TODO: API 서버 배포시 수정
     async function fetchData() {