up

sueszli · Dec 30, 2024 · 4ea6e1a · 4ea6e1a
1 parent 5f71d0c
commit 4ea6e1a
Show file tree

Hide file tree

Showing 7 changed files with 92 additions and 17 deletions.
diff --git a/README b/README
@@ -1 +1,53 @@
+rethinking adversarial examples
+
 check it out: https://sueszli.github.io/thesis/thesis.pdf
+
+                                   ..
+                                .d&9&'
+                             .o$6H&'                        o\.
+                           .+S$kM'                          ?&Rb.
+                         _+S}R&H?                           `\&9Hb.
+                        +Z$&&&H&D?.o/p1&S6H&&D#>\\           !k&HHH,
+                       |Z$Z&&16&FFR/&&Sk6kMS&9H&HM&b\         H&&&HM:
+                       R$SZ&&&ZZ6$?F$$S$S$&99$F6H&6H&H_   ..\d&&&M9HM
+                       6$S&FS&>':/S$SF/d$S$&F6&&&H&&SH&pp&&F1Sk6R6M9M:
+                       +$DF1?  -i&$Z'-`.:\Z&&&&1SkM&1&&Z$$Z$k&kMH&HHM'
+                       `&F&' . \Z$?  - :-)&&>/&%kkkR&&&&&Z&H9H9HHRM9'
+                       .R&-... {%?  .` - %&$]SFF&&D&$RH16&9&H9DRH6H'
+                       |&:?|H::Fb.>\-.'.'?T&/$&$k&SR6H&M1RM&6MH9M>
+                      .R&9&M&$Z6R.6H: ..::1&(k&1&&1kM&D6M&6HDM&"
+                     .dSFH9&R61&&H9&r ..`|HZ$1SkH&$HH6HHH&M*"'
+                   .d&/$$H&HZ$19kM&6? . :$F$&/$$H&H&9H96M9|
+           ..      {$/&$&$S$$S\$RH9?.`-;/Z$$S$&ZkHH&DHH&MHk
+            \|d_   &S$&$&$?k&%FFk6$c//}kH$&Z&$F11&SMH9&9H9|
+             |7Z&. T$\$&]&&$H$9&1&}Z$k&?$\1%$&$&9H1&9H9&6M-
+        /:/< `  |9, `$$SD6S[&Z$S$/S}$$]SF$/$S&&$&H&6HHHHH'
+        `?<&>,.  >}  `"/$H&$ZF14%&$S$&$S&H$kFkR&kR6HH&6"
+  ... .  ` '<#HqdH'      ""<&Z$&Z$}&&SH&&&S&&&FRH&6HH"
+   |i???    .?9H&Db..o+<vo. ''<$RH&&&&H$&H$9&1&HHH9'
+     "d$HZ$[bq"-"#H&R/$$S&H.   ``$FH&1&66D9&&$&DRM!
+        `''''     "RDZ$&&9&      {SFS$F&$&&S$F&9HHc
+                  &k&%F&&H#&Do_o&Z$F$Z&$Z&$ZF1%kH6M|
+                 |1?$S$&9&&H&&RkR&&$F&$S&F&&&SkH&&RM.
+                  `<&&H1&FRHRH9&$Z}F1Z&FkFkM&1&&HH&M?
+                     &\&F6&9&H6R$$]%}F1Z&Z&9H&&6HH96M.
+                         `<H9$&S/Zk&SF$&&1&1&&9&M&RHM?
+                           $9$&$Z$&$S&&F6FM$M&kH6HH&M!
+                            $RF6&$FkFR&H&6HH&R6M9RHM'
+                           ,$&&kH&1&H&H&HH9HHH&H6HM9.
+                           :}&\1ZFkH&H6HFM&MHHRM&MHHL
+                           :k&$?FS$H&&&SFkM&9&RMM&HRM,
+                            `kZ$S1$%&$&S6&&&196HHHHH6H
+                             `>F&{S$k&Z&$R&H1DkM9RM&HHk.
+                             -bH&&1H1S&FM&H&HH96HRHH6M9Hb.
+                              &S$<1&&&6&$6H&9&&&H&[#9H&M&Mb\.
+                           .,</$\$$$FR$&$$9&11&14&M,`?9HRHDRM&#o\_.
+           .._-,..-~<c&q%S&$$%$[&$&&S}}b\\\&&&HS|H&H.  `"\#RHRH96RMH#qbo\_
+        ,\'': . .: \??H$*/^`:`^d$&&$$&H$FR&Z$$&FR&9&b       ``"`"#>H&9H96HM&Dbo__
+       -\:'/.:.:|:(<$' ... . .:-;;1&]%k&&SF&Fk&FM&1kRL               '`"""*#>RH&RHHqo_
+        `/:,:.\.:??$-.:.,:;,:?<%?/J&1$&91&FkM&&&H&11F*                        ""+&9&FMb
+            ````'""`..::.::\?$$7$S$FFF1&&1Z&&&S<+'''                     |#.      `H9HH|
+                     ``-\\\||?<d>$*+>?++^'''`'                        .o$6$b ._,vd&HHHM'
+                                                                 ._o+&9$&1&$RRMHRHRH*'
+                                                             .#91SD&Z&k6k6D6HH&6M&HH=
+                                                              `'''"""`` ` `
diff --git a/figures/10_10_10_False.png b/figures/10_10_10_False.png
diff --git a/figures/3_2_2_False.png b/figures/3_2_2_False.png
diff --git a/figures/4_4_5_True.png b/figures/4_4_5_True.png
diff --git a/figures/6_2_2_True.png b/figures/6_2_2_True.png
diff --git a/thesis.pdf b/thesis.pdf
diff --git a/thesis.tex b/thesis.tex
@@ -966,54 +966,79 @@ \section{Experimental Setup}
 \begin{figure}
 	\centering
 	\includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/self_ensemble_8train_cifar10.pdf}
-	\caption{Training combinations on CIFAR10, with the red dotted line for the ensemble. The X-axis shows layers, the Y-axis shows accuracy, the X-grid shows train configs and the Y-grid shows different attacks.}
+	\caption{Training strategy combinations on CIFAR10, with the red dotted line for the ensemble. The X-axis shows layers, the Y-axis shows accuracy, the X-grid shows train configs and the Y-grid shows different attacks.}
 	\label{fig:8train-cifar10}
 \end{figure}
 
 \begin{figure}
 	\centering
 	\includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/self_ensemble_8train_cifar100.pdf}
-	\caption{Training combinations on CIFAR100, with the red dotted line for the ensemble. The X-axis shows layers, the Y-axis shows accuracy, the X-grid shows train configs and the Y-grid shows different attacks.}
+	\caption{Training strategy combinations on CIFAR100, with the red dotted line for the ensemble. The X-axis shows layers, the Y-axis shows accuracy, the X-grid shows train configs and the Y-grid shows different attacks.}
 	\label{fig:8train-cifar100}
 \end{figure}
 
 \paragraph{Dataset Selection.}
 
 We evaluated on CIFAR-10 only.
 
-Although we implemented the code for CIFAR-10, CIFAR-100 and Imagenette, we only evaluated CIFAR-10 due to time constraints. These constraints were due to the limited availability of GPUs with at least 80 GB memory, which is required by the self-ensembled ResNet model. We were unable to reduce the model's memory requirements, but we did identify compute bottlenecks in the codebase, some of which we also addressed as a pull request to the authors.
+Despite implementing and configuring our evaluation pipeline for 3 datasets (CIFAR-10, CIFAR-100 and Imagenette), we only evaluated CIFAR-10 due to time constraints. These constraints were due to the limited availability of GPUs with at least 80 GB memory, which is required by the self-ensembled ResNet model. We were unable to reduce the model's memory requirements, but we did identify compute bottlenecks in the codebase, some of which we also addressed as a pull request to the authors.
 
-However, as already visible in Figure~\ref{fig:8train-cifar10} vs.\ Figure~\ref{fig:8train-cifar100}, classifying CIFAR-10 is clearly less challenging than other datasets for the self-ensembled ResNet and not representative. Therefore, our results should be taken with a grain of salt and further evaluation on CIFAR-100 and Imagenette is necessary for more conclusive results. But this should not diminish the value of our findings, as they provide a solid foundation and by performing a thorough evaluation.
+However, as visible in Figure~\ref{fig:8train-cifar10} vs.\ Figure~\ref{fig:8train-cifar100}, classifying CIFAR-10 is clearly less challenging than other datasets for the self-ensembled ResNet and this dataset is not representative for the others. Therefore, our results should be taken with a grain of salt and further evaluation is necessary for more conclusive results. This should however not diminish the value of our findings on CIFAR-10 and our methodological contributions.
 
-\paragraph{Adversarial Attack Selection.}
+\paragraph{Attack Selection.}
 
-% PGD, FGSM, masks
-% \footnote{Thanks to Nicholas Carlini for advising us on suitable attacks.}
+We benchmarked against geometric masks, FGSM and PGD.
 
-For the geometric masks we used the same rendering engine library we built. Given the particular effectiveness of the knit mask on the ResNet architecture, we noticed that in addition to the mask opacity, the mask density also played a significant role in the effectiveness of the attack. Therefore, we generalized and parametrized our mask generator with the following parameters: number of sides (3, 4, 6, 10)\footnote{3 sides correspond to a triangle, 4 sides to a square, 6 sides to a hexagon and 10 sides to a decagon.}, number of concentric shapes (1, 2, 3, 4) and colors (True, False). We generated masks for all combinations of these parameters for our experiments.
+The authors use a general benchmarking suite and visualize their attacks against FGSM, conducting only light adversarial training with FGSM. However, after consulting\footnote{Thanks to Nicholas Carlini for his advice.} and manual inspection, we decided to also include Projected Gradient Descent (PGD) attacks in our benchmarking suite.
 
+For the geometric masks, we extended our rendering pipeline from previous experiments. Building on the observation that opacity and density of a mask are generally relevant across all models and the effectiveness of the ``Knit'' mask on the vanilla ResNet architecture (which we also kept, see Figure \ref{fig:attacks-v2}), we parametrized our mask generator with the following parameters: number of sides (3, 4, 6, 10), number of concentric shapes (1, 2, 3, 4) and colors (True, False). We generated masks for all combinations of these parameters for our experiments. Selected examples of these masks are shown in Figure~\ref{fig:attacks-v2}.
+
+\begin{figure}[th]
+	\begin{subfigure}[t]{0.19\textwidth}\centering\includegraphics[width=\linewidth]{figures/3_2_2_False.png}\caption{Triangle mask}\label{fig:atk1}\end{subfigure}
+	\hfill
+	\begin{subfigure}[t]{0.19\textwidth}\centering\includegraphics[width=\linewidth]{figures/4_4_5_True.png}\caption{Square mask}\label{fig:atk2}\end{subfigure}
+	\hfill
+	\begin{subfigure}[t]{0.19\textwidth}\centering\includegraphics[width=\linewidth]{figures/6_2_2_True.png}\caption{Hexagon mask}\label{fig:atk3}\end{subfigure}
+	\hfill
+	\begin{subfigure}[t]{0.19\textwidth}\centering\includegraphics[width=\linewidth]{figures/10_10_10_False.png}\caption{Decagon mask}\label{fig:atk4}\end{subfigure}
+
+	\caption{Selected examples of geometric masks used in our experiments with varying sides, count of shapes per row/column, number of concentric shapes and colors.}
+	\label{fig:attacks-v2}
+\end{figure}
 
 
-\paragraph{Baseline.} We benchmarked a ResNet-50 for comparison.
 
-...
 
 
-\section{Results}
 
 
 \begin{figure}
 	\centering
-	\includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/self_ensemble_fgsm_pgd_cifar10.pdf}
+	\includegraphics[width=1\columnwidth,keepaspectratio]{figures/self_ensemble_mask_exploration_cifar10.pdf}
 	\caption{Wassup.}
-	\label{fig:wassup1}
+	\label{fig:wassup2}
 \end{figure}
 
+
+
+
+
+
+
+
+
+\paragraph{Baseline.} We benchmarked a ResNet-50 for comparison.
+
+...
+
+
+\section{Results}
+
 \begin{figure}
 	\centering
-	\includegraphics[width=1\columnwidth,keepaspectratio]{figures/self_ensemble_mask_exploration_cifar10.pdf}
+	\includegraphics[width=\textwidth,height=\textheight,keepaspectratio]{figures/self_ensemble_fgsm_pgd_cifar10.pdf}
 	\caption{Wassup.}
-	\label{fig:wassup2}
+	\label{fig:wassup1}
 \end{figure}
 
 \begin{figure}
@@ -1032,8 +1057,6 @@ \section{Results}
 
 
 
-
-
 \section{Conclusion}
 
 \paragraph{Outlook.}