diff --git a/contracts/core/Modifier.sol b/contracts/core/Modifier.sol
index aa2b963c..7b5a6c1b 100644
--- a/contracts/core/Modifier.sol
+++ b/contracts/core/Modifier.sol
@@ -74,15 +74,10 @@ abstract contract Modifier is Module, IAvatar, EIP712Signature {
     */
 
     modifier moduleOnly() {
-        bool wasSent = modules[msg.sender] != address(0);
-        bool wasSigned = !wasSent && modules[eip712SignedBy()] != address(0);
-        bool isAuthorized = wasSent || wasSigned;
-
-        if (!isAuthorized) {
-            revert NotAuthorized(msg.sender);
-        }
-
-        if (wasSigned) {
+        if (modules[msg.sender] == address(0)) {
+            if (modules[eip712SignedBy()] == address(0)) {
+                revert NotAuthorized(msg.sender);
+            }
             eip712BumpNonce();
         }