03. preprocessing_unlabeled.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"03. preprocessing_unlabeled.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"metadata":{"id":"qY6o33ztLI7J","colab_type":"code","outputId":"c1e23d9b-5bf1-414e-ddd7-fb09818d6c0e","executionInfo":{"status":"ok","timestamp":1553511582157,"user_tz":-330,"elapsed":1138,"user":{"displayName":"Abhay Tyagi","photoUrl":"","userId":"17391893291666241300"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive/')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"],"name":"stdout"}]},{"metadata":{"id":"-TTSBdy3LQwe","colab_type":"code","colab":{}},"cell_type":"code","source":["import pandas as pd"],"execution_count":0,"outputs":[]},{"metadata":{"id":"U9AjFdIhLZas","colab_type":"code","colab":{}},"cell_type":"code","source":["col_names = ['duration', 'protocol_type', 'service', 'flag', 'src_bytes', \n","             'dst_bytes', 'land', 'wrong_fragment', 'urgent', 'hot', \n","             'num_failed_logins', 'logged_in', 'num_compromised', 'root_shell',\n","             'su_attempted', 'num_root', 'num_file_creations', 'num_shells', \n","             'num_access_files', 'num_outbound_cmds', 'is_host_login', \n","             'is_guest_login', 'count', 'srv_count', 'serror_rate', \n","             'srv_serror_rate', 'rerror_rate', 'srv_rerror_rate', \n","             'same_srv_rate', 'diff_srv_rate', 'srv_diff_host_rate', \n","             'dst_host_count', 'dst_host_srv_count', 'dst_host_same_srv_rate', \n","             'dst_host_diff_srv_rate', 'dst_host_same_src_port_rate',\n","             'dst_host_srv_diff_host_rate', 'dst_host_serror_rate', \n","             'dst_host_srv_serror_rate', 'dst_host_rerror_rate', \n","             'dst_host_srv_rerror_rate']\n","data = pd.read_csv('/content/drive/My Drive/Colab Notebooks/Datasets/PCAPTest.csv',\n","                  header=None, names=col_names)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"rQLZmR-iLieE","colab_type":"code","colab":{}},"cell_type":"code","source":["df = data.dropna()"],"execution_count":0,"outputs":[]},{"metadata":{"id":"ca8w7Xi6Lqs8","colab_type":"code","outputId":"42de65d6-7e35-40df-c74d-0e8f6b35a6b5","executionInfo":{"status":"ok","timestamp":1553511615860,"user_tz":-330,"elapsed":18974,"user":{"displayName":"Abhay Tyagi","photoUrl":"","userId":"17391893291666241300"}},"colab":{"base_uri":"https://localhost:8080/","height":253}},"cell_type":"code","source":["df.head()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>duration</th>\n","      <th>protocol_type</th>\n","      <th>service</th>\n","      <th>flag</th>\n","      <th>src_bytes</th>\n","      <th>dst_bytes</th>\n","      <th>land</th>\n","      <th>wrong_fragment</th>\n","      <th>urgent</th>\n","      <th>hot</th>\n","      <th>...</th>\n","      <th>dst_host_count</th>\n","      <th>dst_host_srv_count</th>\n","      <th>dst_host_same_srv_rate</th>\n","      <th>dst_host_diff_srv_rate</th>\n","      <th>dst_host_same_src_port_rate</th>\n","      <th>dst_host_srv_diff_host_rate</th>\n","      <th>dst_host_serror_rate</th>\n","      <th>dst_host_srv_serror_rate</th>\n","      <th>dst_host_rerror_rate</th>\n","      <th>dst_host_srv_rerror_rate</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>udp</td>\n","      <td>private</td>\n","      <td>SF</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1.0</td>\n","      <td>0.00</td>\n","      <td>1.00</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>udp</td>\n","      <td>private</td>\n","      <td>SF</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>255</td>\n","      <td>254</td>\n","      <td>1.0</td>\n","      <td>0.01</td>\n","      <td>0.00</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>udp</td>\n","      <td>private</td>\n","      <td>SF</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>255</td>\n","      <td>254</td>\n","      <td>1.0</td>\n","      <td>0.01</td>\n","      <td>0.00</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>udp</td>\n","      <td>private</td>\n","      <td>SF</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>255</td>\n","      <td>254</td>\n","      <td>1.0</td>\n","      <td>0.01</td>\n","      <td>0.00</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>udp</td>\n","      <td>private</td>\n","      <td>SF</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>255</td>\n","      <td>254</td>\n","      <td>1.0</td>\n","      <td>0.01</td>\n","      <td>0.01</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>5 rows × 41 columns</p>\n","</div>"],"text/plain":["   duration protocol_type  service flag  src_bytes  dst_bytes  land  \\\n","0         0           udp  private   SF        105        146     0   \n","1         0           udp  private   SF        105        146     0   \n","2         0           udp  private   SF        105        146     0   \n","3         0           udp  private   SF        105        146     0   \n","4         0           udp  private   SF        105        146     0   \n","\n","   wrong_fragment  urgent  hot            ...             dst_host_count  \\\n","0               0       0    0            ...                          1   \n","1               0       0    0            ...                        255   \n","2               0       0    0            ...                        255   \n","3               0       0    0            ...                        255   \n","4               0       0    0            ...                        255   \n","\n","   dst_host_srv_count  dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n","0                   1                     1.0                    0.00   \n","1                 254                     1.0                    0.01   \n","2                 254                     1.0                    0.01   \n","3                 254                     1.0                    0.01   \n","4                 254                     1.0                    0.01   \n","\n","   dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n","0                         1.00                          0.0   \n","1                         0.00                          0.0   \n","2                         0.00                          0.0   \n","3                         0.00                          0.0   \n","4                         0.01                          0.0   \n","\n","   dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n","0                   0.0                       0.0                   0.0   \n","1                   0.0                       0.0                   0.0   \n","2                   0.0                       0.0                   0.0   \n","3                   0.0                       0.0                   0.0   \n","4                   0.0                       0.0                   0.0   \n","\n","   dst_host_srv_rerror_rate  \n","0                       0.0  \n","1                       0.0  \n","2                       0.0  \n","3                       0.0  \n","4                       0.0  \n","\n","[5 rows x 41 columns]"]},"metadata":{"tags":[]},"execution_count":19}]},{"metadata":{"id":"V4KL_TEYLtcW","colab_type":"code","outputId":"f83346c1-7eaf-45e6-e2e4-ea187e7a3b60","executionInfo":{"status":"ok","timestamp":1553511623533,"user_tz":-330,"elapsed":24945,"user":{"displayName":"Abhay Tyagi","photoUrl":"","userId":"17391893291666241300"}},"colab":{"base_uri":"https://localhost:8080/","height":346}},"cell_type":"code","source":["df.describe()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>duration</th>\n","      <th>src_bytes</th>\n","      <th>dst_bytes</th>\n","      <th>land</th>\n","      <th>wrong_fragment</th>\n","      <th>urgent</th>\n","      <th>hot</th>\n","      <th>num_failed_logins</th>\n","      <th>logged_in</th>\n","      <th>num_compromised</th>\n","      <th>...</th>\n","      <th>dst_host_count</th>\n","      <th>dst_host_srv_count</th>\n","      <th>dst_host_same_srv_rate</th>\n","      <th>dst_host_diff_srv_rate</th>\n","      <th>dst_host_same_src_port_rate</th>\n","      <th>dst_host_srv_diff_host_rate</th>\n","      <th>dst_host_serror_rate</th>\n","      <th>dst_host_srv_serror_rate</th>\n","      <th>dst_host_rerror_rate</th>\n","      <th>dst_host_srv_rerror_rate</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>...</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","      <td>2.984154e+06</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>3.720818e+00</td>\n","      <td>7.688802e+02</td>\n","      <td>7.308897e+02</td>\n","      <td>3.015930e-06</td>\n","      <td>4.771872e-04</td>\n","      <td>1.373924e-05</td>\n","      <td>5.270170e-03</td>\n","      <td>2.597051e-04</td>\n","      <td>1.470725e-01</td>\n","      <td>4.903567e-03</td>\n","      <td>...</td>\n","      <td>2.355073e+02</td>\n","      <td>1.991410e+02</td>\n","      <td>7.902420e-01</td>\n","      <td>2.417571e-02</td>\n","      <td>5.676274e-01</td>\n","      <td>4.283301e-03</td>\n","      <td>6.000331e-02</td>\n","      <td>5.988593e-02</td>\n","      <td>1.456255e-01</td>\n","      <td>1.459071e-01</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>2.439534e+02</td>\n","      <td>4.555130e+04</td>\n","      <td>3.991422e+04</td>\n","      <td>1.736641e-03</td>\n","      <td>3.568144e-02</td>\n","      <td>6.098877e-03</td>\n","      <td>3.037666e-01</td>\n","      <td>1.710198e-02</td>\n","      <td>3.541782e-01</td>\n","      <td>1.480858e+00</td>\n","      <td>...</td>\n","      <td>6.033789e+01</td>\n","      <td>1.008801e+02</td>\n","      <td>3.917469e-01</td>\n","      <td>9.541677e-02</td>\n","      <td>4.894808e-01</td>\n","      <td>3.412028e-02</td>\n","      <td>2.348805e-01</td>\n","      <td>2.363297e-01</td>\n","      <td>3.497601e-01</td>\n","      <td>3.519739e-01</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>...</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>0.000000e+00</td>\n","      <td>1.050000e+02</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>...</td>\n","      <td>2.550000e+02</td>\n","      <td>2.480000e+02</td>\n","      <td>9.900000e-01</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>0.000000e+00</td>\n","      <td>5.200000e+02</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>...</td>\n","      <td>2.550000e+02</td>\n","      <td>2.550000e+02</td>\n","      <td>1.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>1.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>0.000000e+00</td>\n","      <td>1.032000e+03</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>...</td>\n","      <td>2.550000e+02</td>\n","      <td>2.550000e+02</td>\n","      <td>1.000000e+00</td>\n","      <td>1.000000e-02</td>\n","      <td>1.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","      <td>0.000000e+00</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>6.636600e+04</td>\n","      <td>6.282565e+07</td>\n","      <td>3.231770e+07</td>\n","      <td>1.000000e+00</td>\n","      <td>3.000000e+00</td>\n","      <td>6.000000e+00</td>\n","      <td>2.330000e+02</td>\n","      <td>5.000000e+00</td>\n","      <td>1.000000e+00</td>\n","      <td>9.420000e+02</td>\n","      <td>...</td>\n","      <td>2.550000e+02</td>\n","      <td>2.550000e+02</td>\n","      <td>1.000000e+00</td>\n","      <td>1.000000e+00</td>\n","      <td>1.000000e+00</td>\n","      <td>1.000000e+00</td>\n","      <td>1.000000e+00</td>\n","      <td>1.000000e+00</td>\n","      <td>1.000000e+00</td>\n","      <td>1.000000e+00</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8 rows × 38 columns</p>\n","</div>"],"text/plain":["           duration     src_bytes     dst_bytes          land  wrong_fragment  \\\n","count  2.984154e+06  2.984154e+06  2.984154e+06  2.984154e+06    2.984154e+06   \n","mean   3.720818e+00  7.688802e+02  7.308897e+02  3.015930e-06    4.771872e-04   \n","std    2.439534e+02  4.555130e+04  3.991422e+04  1.736641e-03    3.568144e-02   \n","min    0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00    0.000000e+00   \n","25%    0.000000e+00  1.050000e+02  0.000000e+00  0.000000e+00    0.000000e+00   \n","50%    0.000000e+00  5.200000e+02  0.000000e+00  0.000000e+00    0.000000e+00   \n","75%    0.000000e+00  1.032000e+03  0.000000e+00  0.000000e+00    0.000000e+00   \n","max    6.636600e+04  6.282565e+07  3.231770e+07  1.000000e+00    3.000000e+00   \n","\n","             urgent           hot  num_failed_logins     logged_in  \\\n","count  2.984154e+06  2.984154e+06       2.984154e+06  2.984154e+06   \n","mean   1.373924e-05  5.270170e-03       2.597051e-04  1.470725e-01   \n","std    6.098877e-03  3.037666e-01       1.710198e-02  3.541782e-01   \n","min    0.000000e+00  0.000000e+00       0.000000e+00  0.000000e+00   \n","25%    0.000000e+00  0.000000e+00       0.000000e+00  0.000000e+00   \n","50%    0.000000e+00  0.000000e+00       0.000000e+00  0.000000e+00   \n","75%    0.000000e+00  0.000000e+00       0.000000e+00  0.000000e+00   \n","max    6.000000e+00  2.330000e+02       5.000000e+00  1.000000e+00   \n","\n","       num_compromised            ...             dst_host_count  \\\n","count     2.984154e+06            ...               2.984154e+06   \n","mean      4.903567e-03            ...               2.355073e+02   \n","std       1.480858e+00            ...               6.033789e+01   \n","min       0.000000e+00            ...               0.000000e+00   \n","25%       0.000000e+00            ...               2.550000e+02   \n","50%       0.000000e+00            ...               2.550000e+02   \n","75%       0.000000e+00            ...               2.550000e+02   \n","max       9.420000e+02            ...               2.550000e+02   \n","\n","       dst_host_srv_count  dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n","count        2.984154e+06            2.984154e+06            2.984154e+06   \n","mean         1.991410e+02            7.902420e-01            2.417571e-02   \n","std          1.008801e+02            3.917469e-01            9.541677e-02   \n","min          0.000000e+00            0.000000e+00            0.000000e+00   \n","25%          2.480000e+02            9.900000e-01            0.000000e+00   \n","50%          2.550000e+02            1.000000e+00            0.000000e+00   \n","75%          2.550000e+02            1.000000e+00            1.000000e-02   \n","max          2.550000e+02            1.000000e+00            1.000000e+00   \n","\n","       dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n","count                 2.984154e+06                 2.984154e+06   \n","mean                  5.676274e-01                 4.283301e-03   \n","std                   4.894808e-01                 3.412028e-02   \n","min                   0.000000e+00                 0.000000e+00   \n","25%                   0.000000e+00                 0.000000e+00   \n","50%                   1.000000e+00                 0.000000e+00   \n","75%                   1.000000e+00                 0.000000e+00   \n","max                   1.000000e+00                 1.000000e+00   \n","\n","       dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n","count          2.984154e+06              2.984154e+06          2.984154e+06   \n","mean           6.000331e-02              5.988593e-02          1.456255e-01   \n","std            2.348805e-01              2.363297e-01          3.497601e-01   \n","min            0.000000e+00              0.000000e+00          0.000000e+00   \n","25%            0.000000e+00              0.000000e+00          0.000000e+00   \n","50%            0.000000e+00              0.000000e+00          0.000000e+00   \n","75%            0.000000e+00              0.000000e+00          0.000000e+00   \n","max            1.000000e+00              1.000000e+00          1.000000e+00   \n","\n","       dst_host_srv_rerror_rate  \n","count              2.984154e+06  \n","mean               1.459071e-01  \n","std                3.519739e-01  \n","min                0.000000e+00  \n","25%                0.000000e+00  \n","50%                0.000000e+00  \n","75%                0.000000e+00  \n","max                1.000000e+00  \n","\n","[8 rows x 38 columns]"]},"metadata":{"tags":[]},"execution_count":20}]},{"metadata":{"id":"KW0AjMhePNhl","colab_type":"code","outputId":"9515d1fd-8068-4837-b7b2-e49ac2145b94","executionInfo":{"status":"ok","timestamp":1553511623534,"user_tz":-330,"elapsed":22369,"user":{"displayName":"Abhay Tyagi","photoUrl":"","userId":"17391893291666241300"}},"colab":{"base_uri":"https://localhost:8080/","height":799}},"cell_type":"code","source":["df.info()"],"execution_count":0,"outputs":[{"output_type":"stream","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 2984154 entries, 0 to 2984153\n","Data columns (total 41 columns):\n","duration                       int64\n","protocol_type                  object\n","service                        object\n","flag                           object\n","src_bytes                      int64\n","dst_bytes                      int64\n","land                           int64\n","wrong_fragment                 int64\n","urgent                         int64\n","hot                            int64\n","num_failed_logins              int64\n","logged_in                      int64\n","num_compromised                int64\n","root_shell                     int64\n","su_attempted                   int64\n","num_root                       int64\n","num_file_creations             int64\n","num_shells                     int64\n","num_access_files               int64\n","num_outbound_cmds              int64\n","is_host_login                  int64\n","is_guest_login                 int64\n","count                          int64\n","srv_count                      int64\n","serror_rate                    float64\n","srv_serror_rate                float64\n","rerror_rate                    float64\n","srv_rerror_rate                float64\n","same_srv_rate                  float64\n","diff_srv_rate                  float64\n","srv_diff_host_rate             float64\n","dst_host_count                 int64\n","dst_host_srv_count             int64\n","dst_host_same_srv_rate         float64\n","dst_host_diff_srv_rate         float64\n","dst_host_same_src_port_rate    float64\n","dst_host_srv_diff_host_rate    float64\n","dst_host_serror_rate           float64\n","dst_host_srv_serror_rate       float64\n","dst_host_rerror_rate           float64\n","dst_host_srv_rerror_rate       float64\n","dtypes: float64(15), int64(23), object(3)\n","memory usage: 956.2+ MB\n"],"name":"stdout"}]},{"metadata":{"id":"F8h0eVrFPT2Z","colab_type":"code","colab":{}},"cell_type":"code","source":["df['protocol_type'] = df['protocol_type'].replace('udp', '0')\n","df['protocol_type'] = df['protocol_type'].replace('tcp', '1')\n","df['protocol_type'] = df['protocol_type'].replace('icmp', '2')"],"execution_count":0,"outputs":[]},{"metadata":{"id":"aIRG5oubPXGh","colab_type":"code","colab":{}},"cell_type":"code","source":["df['service'] = df['service'].replace('private', '0')\n","df['service'] = df['service'].replace('domain_u', '1')\n","df['service'] = df['service'].replace('http', '2')\n","df['service'] = df['service'].replace('smtp', '3')\n","df['service'] = df['service'].replace('ftp_data', '4')\n","df['service'] = df['service'].replace('ftp', '5')\n","df['service'] = df['service'].replace('eco_i', '6')\n","df['service'] = df['service'].replace('other', '7')\n","df['service'] = df['service'].replace('auth', '8')\n","df['service'] = df['service'].replace('ecr_i', '9')\n","df['service'] = df['service'].replace('IRC', '10')\n","df['service'] = df['service'].replace('X11', '11')\n","df['service'] = df['service'].replace('finger', '12')\n","df['service'] = df['service'].replace('time', '13')\n","df['service'] = df['service'].replace('domain', '14')\n","df['service'] = df['service'].replace('telnet', '15')\n","df['service'] = df['service'].replace('pop_3', '16')\n","df['service'] = df['service'].replace('ldap', '17')\n","df['service'] = df['service'].replace('login', '18')\n","df['service'] = df['service'].replace('name', '19')\n","df['service'] = df['service'].replace('ntp_u', '20')\n","df['service'] = df['service'].replace('http_443', '21')\n","df['service'] = df['service'].replace('sunrpc', '22')\n","df['service'] = df['service'].replace('printer', '23')\n","df['service'] = df['service'].replace('systat', '24')\n","df['service'] = df['service'].replace('tim_i', '25')\n","df['service'] = df['service'].replace('netstat', '26')\n","df['service'] = df['service'].replace('remote_job', '27')\n","df['service'] = df['service'].replace('link', '28')\n","df['service'] = df['service'].replace('urp_i', '29')\n","df['service'] = df['service'].replace('sql_net', '30')\n","df['service'] = df['service'].replace('bgp', '31')\n","df['service'] = df['service'].replace('pop_2', '32')\n","df['service'] = df['service'].replace('tftp_u', '33')\n","df['service'] = df['service'].replace('uucp', '34')\n","df['service'] = df['service'].replace('imap4', '35')\n","df['service'] = df['service'].replace('pm_dump', '36')\n","df['service'] = df['service'].replace('nnsp', '37')\n","df['service'] = df['service'].replace('courier', '38')\n","df['service'] = df['service'].replace('daytime', '39')\n","df['service'] = df['service'].replace('iso_tsap', '40')\n","df['service'] = df['service'].replace('echo', '41')\n","df['service'] = df['service'].replace('discard', '42')\n","df['service'] = df['service'].replace('ssh', '43')\n","df['service'] = df['service'].replace('whois', '44')\n","df['service'] = df['service'].replace('mtp', '45')\n","df['service'] = df['service'].replace('gopher', '46')\n","df['service'] = df['service'].replace('rje', '47')\n","df['service'] = df['service'].replace('ctf', '48')\n","df['service'] = df['service'].replace('supdup', '49')\n","df['service'] = df['service'].replace('hostnames', '50')\n","df['service'] = df['service'].replace('csnet_ns', '51')\n","df['service'] = df['service'].replace('uucp_path', '52')\n","df['service'] = df['service'].replace('nntp', '53')\n","df['service'] = df['service'].replace('netbios_ns', '54')\n","df['service'] = df['service'].replace('netbios_dgm', '55')\n","df['service'] = df['service'].replace('netbios_ssn', '56')\n","df['service'] = df['service'].replace('vmnet', '57')\n","df['service'] = df['service'].replace('Z39_50', '58')\n","df['service'] = df['service'].replace('exec', '59')\n","df['service'] = df['service'].replace('shell', '60')\n","df['service'] = df['service'].replace('efs', '61')\n","df['service'] = df['service'].replace('klogin', '62')\n","df['service'] = df['service'].replace('kshell', '63')\n","df['service'] = df['service'].replace('icmp', '64')\n","#df['service'] = df['service'].replace('harvest', '64')\n","#df['service'] = df['service'].replace('http_2784', '64')\n","#df['service'] = df['service'].replace('http_8001', '64')\n","#df['service'] = df['service'].replace('urh_i', '64')\n","#df['service'] = df['service'].replace('aol', '64')\n","df = df[df.service != 'harvest']\n","df = df[df.service != 'http_2784']\n","df = df[df.service != 'http_8001']\n","df = df[df.service != 'urh_i']\n","df = df[df.service != 'aol']"],"execution_count":0,"outputs":[]},{"metadata":{"id":"wC6E_T4pPauf","colab_type":"code","colab":{}},"cell_type":"code","source":["df['flag'] = df['flag'].replace('SF', '0')\n","df['flag'] = df['flag'].replace('SH', '1')\n","df['flag'] = df['flag'].replace('RSTR', '2')\n","df['flag'] = df['flag'].replace('REJ', '3')\n","df['flag'] = df['flag'].replace('S0', '4')\n","df['flag'] = df['flag'].replace('S1', '5')\n","df['flag'] = df['flag'].replace('S2', '6')\n","df['flag'] = df['flag'].replace('S3', '7')\n","df['flag'] = df['flag'].replace('RSTO', '8')\n","df['flag'] = df['flag'].replace('RSTOS0', '9')\n","df['flag'] = df['flag'].replace('OTH', '10')"],"execution_count":0,"outputs":[]},{"metadata":{"id":"S5PDmYLjPdee","colab_type":"code","outputId":"bf56fe53-a30d-463d-cc08-d2bc2d82ac8f","executionInfo":{"status":"ok","timestamp":1553511827682,"user_tz":-330,"elapsed":798,"user":{"displayName":"Abhay Tyagi","photoUrl":"","userId":"17391893291666241300"}},"colab":{"base_uri":"https://localhost:8080/","height":253}},"cell_type":"code","source":["df.head()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>duration</th>\n","      <th>protocol_type</th>\n","      <th>service</th>\n","      <th>flag</th>\n","      <th>src_bytes</th>\n","      <th>dst_bytes</th>\n","      <th>land</th>\n","      <th>wrong_fragment</th>\n","      <th>urgent</th>\n","      <th>hot</th>\n","      <th>...</th>\n","      <th>dst_host_count</th>\n","      <th>dst_host_srv_count</th>\n","      <th>dst_host_same_srv_rate</th>\n","      <th>dst_host_diff_srv_rate</th>\n","      <th>dst_host_same_src_port_rate</th>\n","      <th>dst_host_srv_diff_host_rate</th>\n","      <th>dst_host_serror_rate</th>\n","      <th>dst_host_srv_serror_rate</th>\n","      <th>dst_host_rerror_rate</th>\n","      <th>dst_host_srv_rerror_rate</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>1</td>\n","      <td>1</td>\n","      <td>1.0</td>\n","      <td>0.00</td>\n","      <td>1.00</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>255</td>\n","      <td>254</td>\n","      <td>1.0</td>\n","      <td>0.01</td>\n","      <td>0.00</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>255</td>\n","      <td>254</td>\n","      <td>1.0</td>\n","      <td>0.01</td>\n","      <td>0.00</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>255</td>\n","      <td>254</td>\n","      <td>1.0</td>\n","      <td>0.01</td>\n","      <td>0.00</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>105</td>\n","      <td>146</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>255</td>\n","      <td>254</td>\n","      <td>1.0</td>\n","      <td>0.01</td>\n","      <td>0.01</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>5 rows × 41 columns</p>\n","</div>"],"text/plain":["   duration protocol_type service flag  src_bytes  dst_bytes  land  \\\n","0         0             0       0    0        105        146     0   \n","1         0             0       0    0        105        146     0   \n","2         0             0       0    0        105        146     0   \n","3         0             0       0    0        105        146     0   \n","4         0             0       0    0        105        146     0   \n","\n","   wrong_fragment  urgent  hot            ...             dst_host_count  \\\n","0               0       0    0            ...                          1   \n","1               0       0    0            ...                        255   \n","2               0       0    0            ...                        255   \n","3               0       0    0            ...                        255   \n","4               0       0    0            ...                        255   \n","\n","   dst_host_srv_count  dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n","0                   1                     1.0                    0.00   \n","1                 254                     1.0                    0.01   \n","2                 254                     1.0                    0.01   \n","3                 254                     1.0                    0.01   \n","4                 254                     1.0                    0.01   \n","\n","   dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n","0                         1.00                          0.0   \n","1                         0.00                          0.0   \n","2                         0.00                          0.0   \n","3                         0.00                          0.0   \n","4                         0.01                          0.0   \n","\n","   dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n","0                   0.0                       0.0                   0.0   \n","1                   0.0                       0.0                   0.0   \n","2                   0.0                       0.0                   0.0   \n","3                   0.0                       0.0                   0.0   \n","4                   0.0                       0.0                   0.0   \n","\n","   dst_host_srv_rerror_rate  \n","0                       0.0  \n","1                       0.0  \n","2                       0.0  \n","3                       0.0  \n","4                       0.0  \n","\n","[5 rows x 41 columns]"]},"metadata":{"tags":[]},"execution_count":26}]},{"metadata":{"id":"_LfU7buuPglB","colab_type":"code","colab":{}},"cell_type":"code","source":["df[\"protocol_type\"] = pd.to_numeric(df[\"protocol_type\"])\n","df[\"service\"] = pd.to_numeric(df[\"service\"])\n","df[\"flag\"]= pd.to_numeric(df[\"flag\"])"],"execution_count":0,"outputs":[]},{"metadata":{"id":"zpu-cE_4Pn52","colab_type":"code","outputId":"f6fcfb7d-4ca3-4c82-cd21-79d4bbd66c6a","executionInfo":{"status":"ok","timestamp":1553511839246,"user_tz":-330,"elapsed":1536,"user":{"displayName":"Abhay Tyagi","photoUrl":"","userId":"17391893291666241300"}},"colab":{"base_uri":"https://localhost:8080/","height":799}},"cell_type":"code","source":["df.info()"],"execution_count":0,"outputs":[{"output_type":"stream","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 2984145 entries, 0 to 2984153\n","Data columns (total 41 columns):\n","duration                       int64\n","protocol_type                  int64\n","service                        int64\n","flag                           int64\n","src_bytes                      int64\n","dst_bytes                      int64\n","land                           int64\n","wrong_fragment                 int64\n","urgent                         int64\n","hot                            int64\n","num_failed_logins              int64\n","logged_in                      int64\n","num_compromised                int64\n","root_shell                     int64\n","su_attempted                   int64\n","num_root                       int64\n","num_file_creations             int64\n","num_shells                     int64\n","num_access_files               int64\n","num_outbound_cmds              int64\n","is_host_login                  int64\n","is_guest_login                 int64\n","count                          int64\n","srv_count                      int64\n","serror_rate                    float64\n","srv_serror_rate                float64\n","rerror_rate                    float64\n","srv_rerror_rate                float64\n","same_srv_rate                  float64\n","diff_srv_rate                  float64\n","srv_diff_host_rate             float64\n","dst_host_count                 int64\n","dst_host_srv_count             int64\n","dst_host_same_srv_rate         float64\n","dst_host_diff_srv_rate         float64\n","dst_host_same_src_port_rate    float64\n","dst_host_srv_diff_host_rate    float64\n","dst_host_serror_rate           float64\n","dst_host_srv_serror_rate       float64\n","dst_host_rerror_rate           float64\n","dst_host_srv_rerror_rate       float64\n","dtypes: float64(15), int64(26)\n","memory usage: 956.2 MB\n"],"name":"stdout"}]},{"metadata":{"id":"sG0Xsu-JPw66","colab_type":"code","colab":{}},"cell_type":"code","source":["df.to_csv(\"/content/drive/My Drive/Colab Notebooks/Datasets/clean_unlabeled.csv\", index=False)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"i5faP5ZCQx0X","colab_type":"code","colab":{}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]}]}