diff --git a/.changeset/bright-snakes-fly.md b/.changeset/bright-snakes-fly.md
new file mode 100644
index 0000000..6d2b411
--- /dev/null
+++ b/.changeset/bright-snakes-fly.md
@@ -0,0 +1,5 @@
+---
+"@modern-js/swc-plugins": minor
+---
+
+refactor: remove auto detect esmodule
diff --git a/crates/plugin_lock_corejs_version/tests/fixtures/lock-corejs/cjs/expected.js b/crates/plugin_lock_corejs_version/tests/fixtures/lock-corejs/cjs/expected.js
index 0fe3c0c..2c438f0 100644
--- a/crates/plugin_lock_corejs_version/tests/fixtures/lock-corejs/cjs/expected.js
+++ b/crates/plugin_lock_corejs_version/tests/fixtures/lock-corejs/cjs/expected.js
@@ -1,4 +1,3 @@
-"use strict";
 var _object_spread = require("@@swc/_/_object_spread");
 require("@@corejs/modules/es.object.to-string.js");
 require("@@corejs/modules/es.promise.js");
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/bugfix-169/expected-remove.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/bugfix-169/expected-remove.js
index b0b4ed4..d58b5f1 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/bugfix-169/expected-remove.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/bugfix-169/expected-remove.js
@@ -1,4 +1,3 @@
-"use strict";
 class Test extends React.Component {
   
 }
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-assign-property/expected-remove.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-assign-property/expected-remove.js
index a2ef304..bb798a0 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-assign-property/expected-remove.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-assign-property/expected-remove.js
@@ -1,4 +1,3 @@
-"use strict";
 class Foo1 extends React.Component {
   render() {}
 
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-assign-property/expected-wrap.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-assign-property/expected-wrap.js
index c1d8ec0..089e7c3 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-assign-property/expected-wrap.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-assign-property/expected-wrap.js
@@ -1,4 +1,3 @@
-"use strict";
 class Foo1 extends React.Component {
     render() {}
 }
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-component/expected-remove.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-component/expected-remove.js
index be3aaaf..35a1308 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-component/expected-remove.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-component/expected-remove.js
@@ -1,4 +1,3 @@
-"use strict";
 class Foo1 extends Component {
   render() {}
 
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-component/expected-wrap.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-component/expected-wrap.js
index 853ce0c..de6f0e5 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-component/expected-wrap.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-component/expected-wrap.js
@@ -1,9 +1,8 @@
-"use strict";
-const _define_property = require("@swc/helpers/_/_define_property");
+import { _ as _define_property } from "@swc/helpers/_/_define_property";
 class Foo1 extends Component {
     render() {}
 }
-_define_property._(Foo1, "propTypes", process.env.NODE_ENV !== "production" ? {
+_define_property(Foo1, "propTypes", process.env.NODE_ENV !== "production" ? {
     foo1: PropTypes.string
 } : {});
 class Foo2 extends Component {
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-global-base-component/expected-remove.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-global-base-component/expected-remove.js
index 0924232..344128b 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-global-base-component/expected-remove.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/es-class-extend-global-base-component/expected-remove.js
@@ -1,11 +1,10 @@
-"use strict";
-const _define_property = require("@swc/helpers/_/_define_property");
+import { _ as _define_property } from "@swc/helpers/_/_define_property";
 class Foo1 extends GlobalComponent {
   render() {}
 
 }
 
-_define_property._(Foo1, "propTypes", {
+_define_property(Foo1, "propTypes", {
   foo1: PropTypes.string
 });
 
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react-assign-property/expected-remove.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react-assign-property/expected-remove.js
index 52a943b..7c426af 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react-assign-property/expected-remove.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react-assign-property/expected-remove.js
@@ -1,4 +1,3 @@
-"use strict";
 var foo = {};
 foo.propTypes = {
   foo: PropTypes.string
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react-assign-property/expected-wrap.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react-assign-property/expected-wrap.js
index 30d32e6..b60c0c7 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react-assign-property/expected-wrap.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react-assign-property/expected-wrap.js
@@ -1,4 +1,3 @@
-"use strict";
 var foo = {};
 foo.propTypes = {
     foo: PropTypes.string
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react/expected-remove.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react/expected-remove.js
index 107cdd5..0929a2f 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react/expected-remove.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/not-react/expected-remove.js
@@ -1,4 +1,3 @@
-"use strict";
 var foo = {
   propTypes: {
     foo: 'bar'
diff --git a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/unsafe-wrap/expected-unsafe-wrap.js b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/unsafe-wrap/expected-unsafe-wrap.js
index 8188239..6b07f39 100644
--- a/crates/plugin_react_utils/tests/fixtures/remove_prop_types/unsafe-wrap/expected-unsafe-wrap.js
+++ b/crates/plugin_react_utils/tests/fixtures/remove_prop_types/unsafe-wrap/expected-unsafe-wrap.js
@@ -1,4 +1,3 @@
-"use strict";
 class Foo1 extends React.Component {
     render() {}
 }
diff --git a/crates/plugin_react_utils/tests/main.rs b/crates/plugin_react_utils/tests/main.rs
index 3a74660..2059d14 100644
--- a/crates/plugin_react_utils/tests/main.rs
+++ b/crates/plugin_react_utils/tests/main.rs
@@ -33,7 +33,8 @@ fn main() {
       };
 
       let swc: Options = serde_json::from_str(r#"{
-        "jsc": { "target": "es2020", "externalHelpers": true, "parser": { "syntax": "ecmascript", "jsx": true } }
+        "jsc": { "target": "es2020", "externalHelpers": true, "parser": { "syntax": "ecmascript", "jsx": true } },
+        "isModule": true
       }"#).unwrap();
       let react_utils = ReactUtilsConfig {
         remove_prop_types: Some(remove_prop_types.clone()),
@@ -64,7 +65,7 @@ fn main() {
 
         let expected_code = read(expected_remove_path);
         tests.push(ExpectedInfo::new(
-          expected_path.to_string_lossy().to_string(),
+          expected_remove_path.to_string_lossy().to_string(),
           expected_code,
           Some(TransformConfig {
             swc: swc.clone(),
diff --git a/crates/plugin_ssr_loader_id/src/lib.rs b/crates/plugin_ssr_loader_id/src/lib.rs
index 497b768..4c88985 100644
--- a/crates/plugin_ssr_loader_id/src/lib.rs
+++ b/crates/plugin_ssr_loader_id/src/lib.rs
@@ -342,7 +342,6 @@ mod test {
             filename: "/root/a.js".into(),
             cwd: PathBuf::from("/root"),
             config_hash: None,
-            is_source_esm: true,
           }),
         )
       },
diff --git a/crates/swc_plugins_collection/src/pass.rs b/crates/swc_plugins_collection/src/pass.rs
index 867211d..7ff820b 100644
--- a/crates/swc_plugins_collection/src/pass.rs
+++ b/crates/swc_plugins_collection/src/pass.rs
@@ -3,10 +3,9 @@ use std::{path::Path, sync::Arc};
 use modularize_imports::{modularize_imports, Config as ModularizedConfig};
 use plugin_config_routes::plugin_config_routes;
 use plugin_lock_corejs_version::lock_corejs_version;
-use plugin_remove_es_module_mark::remove_es_module_mark;
 use plugin_ssr_loader_id::plugin_ssr_loader_id;
 use swc_core::{
-  base::config::{ModuleConfig, Options},
+  base::config::Options,
   common::{chain, comments::Comments, pass::Either, FileName},
   ecma::visit::Fold,
   ecma::{transforms::base::pass::noop, visit::as_folder},
@@ -110,7 +109,7 @@ pub fn internal_transform_before_pass<'a>(
 
 pub fn internal_transform_after_pass<'a>(
   extensions: &Extensions,
-  swc_config: &Options,
+  _swc_config: &Options,
   plugin_context: Arc<PluginContext>,
 ) -> impl Fold + 'a {
   let lock_core_js = if let Some(config) = &extensions.lock_corejs_version {
@@ -128,13 +127,7 @@ pub fn internal_transform_after_pass<'a>(
     Either::Right(noop())
   };
 
-  let remove_es_module_mark = if let Some(ModuleConfig::CommonJs(_)) = swc_config.config.module && !plugin_context.is_source_esm {
-    Either::Left(remove_es_module_mark())
-  } else {
-    Either::Right(noop())
-  };
-
-  chain!(lock_core_js, remove_es_module_mark, loadable_components)
+  chain!(lock_core_js, loadable_components)
 }
 
 fn plugin_loadable_components<C: Comments>(comments: C) -> impl Fold {
diff --git a/crates/swc_plugins_collection/tests/snapshots/main__plugin-import.snap b/crates/swc_plugins_collection/tests/snapshots/main__plugin-import.snap
index ec682d2..92f6a05 100644
--- a/crates/swc_plugins_collection/tests/snapshots/main__plugin-import.snap
+++ b/crates/swc_plugins_collection/tests/snapshots/main__plugin-import.snap
@@ -2,7 +2,6 @@
 source: crates/swc_plugins_collection/tests/main.rs
 expression: res.code
 ---
-"use strict";
 var _object_spread = require("@swc/helpers/_/_object_spread");
 var a = {};
 var b = _object_spread._({}, a);
diff --git a/crates/swc_plugins_core/src/transform.rs b/crates/swc_plugins_core/src/transform.rs
index ce81e76..5916c6c 100644
--- a/crates/swc_plugins_core/src/transform.rs
+++ b/crates/swc_plugins_core/src/transform.rs
@@ -3,7 +3,7 @@ use std::{path::PathBuf, sync::Arc};
 use anyhow::Result;
 use swc_core::{
   base::{
-    config::{self, ModuleConfig, Options},
+    config::{self, Options},
     try_with_handler, Compiler, HandlerOpts, TransformOutput,
   },
   common::{comments::SingleThreadedComments, errors::ColorConfig, FileName, Mark, GLOBALS},
@@ -15,7 +15,7 @@ use swc_core::{
     // transforms::module::common_js::Config
   },
 };
-use swc_plugins_utils::{is_esm, PluginContext};
+use swc_plugins_utils::PluginContext;
 
 use crate::TransformFn;
 
@@ -75,8 +75,11 @@ where
 
           swc_config.top_level_mark = Some(top_level_mark);
           let comments = SingleThreadedComments::default();
+          let is_module = swc_config
+            .config
+            .is_module
+            .unwrap_or(config::IsModule::Unknown);
 
-          // Need auto detect esm
           let program = compiler.parse_js(
             fm.clone(),
             handler,
@@ -88,28 +91,10 @@ where
                 ..Default::default()
               })
             }),
-            config::IsModule::Bool(true),
+            is_module,
             Some(&comments),
           )?;
 
-          let is_source_esm = is_esm(&program);
-          // Automatic set module config by it's original format
-          if swc_config.config.module.is_none() {
-            swc_config.config.module = Some(if is_source_esm {
-              ModuleConfig::Es6
-            } else {
-              ModuleConfig::CommonJs(
-                // Remove this when `swc_core` public module config API
-                serde_json::from_str(
-                  r#"{
-                    "ignoreDynamic": true
-                  }"#,
-                )
-                .unwrap(),
-              )
-            });
-          }
-
           // TODO comments can be pass to `process_js_with_custom_pass` in next swc version
           let plugin_context = Arc::new(PluginContext {
             cm,
@@ -118,7 +103,6 @@ where
             unresolved_mark,
             comments: comments.clone(),
             config_hash,
-            is_source_esm,
             filename,
             cwd: swc_config.cwd.clone(),
           });
diff --git a/crates/swc_plugins_utils/src/lib.rs b/crates/swc_plugins_utils/src/lib.rs
index 7ac54d3..13c0db7 100644
--- a/crates/swc_plugins_utils/src/lib.rs
+++ b/crates/swc_plugins_utils/src/lib.rs
@@ -642,10 +642,6 @@ pub struct PluginContext {
   pub cwd: PathBuf,
 
   pub config_hash: Option<String>, // This can be used by plugins to do caching
-
-  // Use this to determine if we should remove __esModule mark in pure commonjs module
-  // Remove this when SWC fix https://github.com/swc-project/swc/issues/6500
-  pub is_source_esm: bool,
 }
 
 impl std::fmt::Debug for PluginContext {
@@ -658,7 +654,6 @@ impl std::fmt::Debug for PluginContext {
       .field("filename", &self.filename)
       .field("cwd", &self.cwd)
       .field("config_hash", &self.config_hash)
-      .field("is_source_esm", &self.is_source_esm)
       .finish()
   }
 }