diff --git a/Cargo.lock b/Cargo.lock index 1c9df20..2b02a18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -911,7 +911,9 @@ dependencies = [ "log", "pyo3", "pyo3-log", + "pythonize", "rquest", + "serde_json", "tokio", ] @@ -1016,6 +1018,16 @@ dependencies = [ "syn", ] +[[package]] +name = "pythonize" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcf491425978bd889015d5430f6473d91bdfa2097262f1e731aadcf6c2113e" +dependencies = [ + "pyo3", + "serde", +] + [[package]] name = "quote" version = "1.0.37" @@ -1121,9 +1133,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rh2" -version = "0.3.50" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5857faed635b86543bc5fa925e7f6719748b3072df253ba5016e61aeffc56dd2" +checksum = "3b0a13d08a2e245517ed50c4fb93bf6ec10f4fc7ea0fbd5193ff26acda084569" dependencies = [ "atomic-waker", "bytes", @@ -1140,9 +1152,9 @@ dependencies = [ [[package]] name = "rhyper" -version = "0.14.51" +version = "0.14.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c270594a5c719889552d795ee0410e00f76ef945ba7babb7fa8e893e18e79632" +checksum = "84b3ef81ea5e8f7ddeef845464b653c1b736eafc787945be9c84e4efff6d33a5" dependencies = [ "bytes", "futures-channel", @@ -1164,9 +1176,9 @@ dependencies = [ [[package]] name = "rquest" -version = "0.26.2" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa610f62e221cb26f7f2f95bd82e2119fd90bc04cf3cf6a1a56670ed98cd3d15" +checksum = "7013d995499eefda3fd60ee7f6f49ffbe69315560b0265119e14dfe004489667" dependencies = [ "antidote", "async-compression", @@ -1193,6 +1205,7 @@ dependencies = [ "rboring-sys", "rhyper", "serde", + "serde_json", "serde_urlencoded", "system-configuration", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 4c53fe5..60a2323 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,9 +16,10 @@ pyo3 = { version = "0.22", features = ["extension-module", "abi3-py38", "indexma anyhow = "1" log = "0.4" pyo3-log = "0.11" -rquest = { version = "0.26", features = [ +rquest = { version = "0.27", features = [ "cookies", "multipart", + "json", "socks", "gzip", "brotli", @@ -31,6 +32,8 @@ indexmap = { version = "2", features = ["serde"] } tokio = { version = "1", features = ["full"] } html2text = "0.13" bytes = "1" +pythonize = "0.22" +serde_json = "1" [profile.release] codegen-units = 1 diff --git a/src/lib.rs b/src/lib.rs index 0835ce1..3aa0a54 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,19 +10,20 @@ use indexmap::IndexMap; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict}; +use pythonize::depythonize; use rquest::boring::x509::{store::X509StoreBuilder, X509}; use rquest::header::{HeaderMap, HeaderName, HeaderValue, COOKIE}; use rquest::tls::Impersonate; use rquest::multipart; use rquest::redirect::Policy; use rquest::Method; +use serde_json::Value; use tokio::runtime::{self, Runtime}; mod response; use response::Response; mod utils; -use utils::{json_dumps, url_encode}; // Tokio global one-thread runtime static RUNTIME: LazyLock = LazyLock::new(|| { @@ -120,9 +121,7 @@ impl Client { } // Client builder - let mut client_builder = rquest::Client::builder() - .enable_ech_grease() - .permute_extensions(); + let mut client_builder = rquest::Client::builder(); // Impersonate if let Some(impersonation_type) = impersonate { @@ -270,14 +269,8 @@ impl Client { }?; let params = params.or(self.params.clone()); let cookies = cookies.or(self.cookies.clone()); - // Converts 'data' (if any) into a URL-encoded string for sending the data as `application/x-www-form-urlencoded` content type. - let data_str = data - .map(|data| url_encode(py, &data.as_unbound())) - .transpose()?; - // Converts 'json' (if any) into a JSON string for sending the data as `application/json` content type. - let json_str = json - .map(|pydict| json_dumps(py, &pydict.as_unbound())) - .transpose()?; + let data_value: Option = data.map(|data| depythonize(&data)).transpose()?; + let json_value: Option = json.map(|json| depythonize(&json)).transpose()?; let auth = auth.or(self.auth.clone()); let auth_bearer = auth_bearer.or(self.auth_bearer.clone()); if auth.is_some() && auth_bearer.is_some() { @@ -325,16 +318,12 @@ impl Client { request_builder = request_builder.body(content); } // Data - if let Some(url_encoded_data) = data_str { - request_builder = request_builder - .header("Content-Type", "application/x-www-form-urlencoded") - .body(url_encoded_data); + if let Some(form_data) = data_value { + request_builder = request_builder.form(&form_data); } // Json - if let Some(json_str) = json_str { - request_builder = request_builder - .header("Content-Type", "application/json") - .body(json_str); + if let Some(json_data) = json_value { + request_builder = request_builder.json(&json_data); } // Files if let Some(files) = files { diff --git a/src/response.rs b/src/response.rs index 32c4cb4..f3a52c7 100644 --- a/src/response.rs +++ b/src/response.rs @@ -1,4 +1,4 @@ -use crate::utils::{get_encoding_from_content, get_encoding_from_headers, json_loads}; +use crate::utils::{get_encoding_from_content, get_encoding_from_headers}; use ahash::RandomState; use anyhow::{anyhow, Result}; use encoding_rs::Encoding; @@ -8,6 +8,8 @@ use html2text::{ }; use indexmap::IndexMap; use pyo3::{prelude::*, types::PyBytes}; +use pythonize::pythonize; +use serde_json::from_slice; /// A struct representing an HTTP response. /// @@ -78,7 +80,8 @@ impl Response { } fn json(&mut self, py: Python) -> Result { - let result = json_loads(py, &self.content)?; + let json_value: serde_json::Value = from_slice(self.content.as_bytes(py))?; + let result = pythonize(py, &json_value).unwrap().into(); Ok(result) } diff --git a/src/utils.rs b/src/utils.rs index 5788662..30a98e8 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,62 +1,7 @@ use std::cmp::min; use ahash::RandomState; -use anyhow::Result; use indexmap::IndexMap; -use pyo3::prelude::*; -use pyo3::sync::GILOnceCell; -use pyo3::types::{PyBool, PyBytes, PyDict}; - -static JSON_DUMPS: GILOnceCell> = GILOnceCell::new(); -static JSON_LOADS: GILOnceCell> = GILOnceCell::new(); -static URLLIB_PARSE_URLENCODE: GILOnceCell> = GILOnceCell::new(); - -/// python json.dumps -pub fn json_dumps(py: Python<'_>, pydict: &Py) -> Result { - let json_dumps = JSON_DUMPS - .get_or_init(py, || { - py.import_bound("json") - .unwrap() - .getattr("dumps") - .unwrap() - .unbind() - }) - .bind(py); - let result = json_dumps.call1((pydict,))?.extract::()?; - Ok(result) -} - -/// python json.loads -pub fn json_loads(py: Python<'_>, content: &Py) -> Result { - let json_loads = JSON_LOADS - .get_or_init(py, || { - py.import_bound("json") - .unwrap() - .getattr("loads") - .unwrap() - .unbind() - }) - .bind(py); - let result = json_loads.call1((content,))?.extract::()?; - Ok(result) -} - -/// python urllib.parse.urlencode -pub fn url_encode(py: Python, pydict: &Py) -> Result { - let urlencode = URLLIB_PARSE_URLENCODE - .get_or_init(py, || { - py.import_bound("urllib.parse") - .unwrap() - .getattr("urlencode") - .unwrap() - .unbind() - }) - .bind(py); - let result: String = urlencode - .call1((pydict, ("doseq", py.get_type_bound::().call1(())?)))? - .extract()?; - Ok(result) -} /// Get encoding from the "Content-Type" header pub fn get_encoding_from_headers( diff --git a/tests/test_client.py b/tests/test_client.py index b231212..c19c94f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,7 +4,7 @@ import primp # type: ignore -def retry(max_retries=5, delay=1): +def retry(max_retries=3, delay=1): def decorator(func): def wrapper(*args, **kwargs): for attempt in range(max_retries): @@ -147,32 +147,6 @@ def test_client_post_data(): assert json_data["form"] == {"key1": "value1", "key2": "value2"} -@retry() -def test_client_post_data2(): - client = primp.Client() - auth_bearer = "bearerXXXXXXXXXXXXXXXXXXXX" - headers = {"X-Test": "test"} - cookies = {"ccc": "ddd", "cccc": "dddd"} - params = {"x": "aaa", "y": "bbb"} - data = {"key1": "value1", "key2": ["value2_1", "value2_2"]} - response = client.post( - "https://httpbin.org/anything", - auth_bearer=auth_bearer, - headers=headers, - cookies=cookies, - params=params, - data=data, - ) - assert response.status_code == 200 - json_data = response.json() - assert json_data["method"] == "POST" - assert json_data["headers"]["X-Test"] == "test" - assert json_data["headers"]["Cookie"] == "ccc=ddd; cccc=dddd" - assert json_data["headers"]["Authorization"] == "Bearer bearerXXXXXXXXXXXXXXXXXXXX" - assert json_data["args"] == {"x": "aaa", "y": "bbb"} - assert json_data["form"] == {"key1": "value1", "key2": ["value2_1", "value2_2"]} - - @retry() def test_client_post_json(): client = primp.Client() @@ -226,16 +200,14 @@ def test_client_post_files(): @retry() -def test_client_impersonate_chrome126(): +def test_client_impersonate_chrome130(): client = primp.Client( - impersonate="chrome_126", + impersonate="chrome_130", ) - response = client.get("https://tls.peet.ws/api/all") + # response = client.get("https://tls.peet.ws/api/all") + response = client.get("https://tls.http.rw/api/clean") assert response.status_code == 200 json_data = response.json() - assert json_data["http_version"] == "h2" - assert json_data["tls"]["ja4"].startswith("t13d") - assert ( - json_data["http2"]["akamai_fingerprint_hash"] - == "90224459f8bf70b7d0a8797eb916dbc9" - ) + assert json_data["ja4"] == "t13d1516h2_8daaf6152771_b1ff8ab2d16f" + assert json_data["akamai_hash"] == "90224459f8bf70b7d0a8797eb916dbc9" + assert json_data["peetprint_hash"] == "b8ce945a4d9a7a9b5b6132e3658fe033" diff --git a/tests/test_defs.py b/tests/test_defs.py index c8f80b3..ce40886 100644 --- a/tests/test_defs.py +++ b/tests/test_defs.py @@ -4,7 +4,7 @@ import primp # type: ignore -def retry(max_retries=5, delay=1): +def retry(max_retries=3, delay=1): def decorator(func): def wrapper(*args, **kwargs): for attempt in range(max_retries): @@ -171,31 +171,6 @@ def test_post_data(): assert json_data["form"] == {"key1": "value1", "key2": "value2"} -@retry() -def test_post_data2(): - auth_bearer = "bearerXXXXXXXXXXXXXXXXXXXX" - headers = {"X-Test": "test"} - cookies = {"ccc": "ddd", "cccc": "dddd"} - params = {"x": "aaa", "y": "bbb"} - data = {"key1": "value1", "key2": ["value2_1", "value2_2"]} - response = primp.post( - "https://httpbin.org/anything", - auth_bearer=auth_bearer, - headers=headers, - cookies=cookies, - params=params, - data=data, - ) - assert response.status_code == 200 - json_data = response.json() - assert json_data["method"] == "POST" - assert json_data["headers"]["X-Test"] == "test" - assert json_data["headers"]["Cookie"] == "ccc=ddd; cccc=dddd" - assert json_data["headers"]["Authorization"] == "Bearer bearerXXXXXXXXXXXXXXXXXXXX" - assert json_data["args"] == {"x": "aaa", "y": "bbb"} - assert json_data["form"] == {"key1": "value1", "key2": ["value2_1", "value2_2"]} - - @retry() def test_post_json(): auth_bearer = "bearerXXXXXXXXXXXXXXXXXXXX" @@ -297,16 +272,14 @@ def test_put(): @retry() -def test_get_impersonate_chrome126(): +def test_get_impersonate_chrome130(): response = primp.get( - "https://tls.peet.ws/api/all", - impersonate="chrome_126", + # "https://tls.peet.ws/api/clean", + "https://tls.http.rw/api/clean", + impersonate="chrome_130", ) assert response.status_code == 200 json_data = response.json() - assert json_data["http_version"] == "h2" - assert json_data["tls"]["ja4"].startswith("t13d") - assert ( - json_data["http2"]["akamai_fingerprint_hash"] - == "90224459f8bf70b7d0a8797eb916dbc9" - ) + assert json_data["ja4"] == "t13d1516h2_8daaf6152771_b1ff8ab2d16f" + assert json_data["akamai_hash"] == "90224459f8bf70b7d0a8797eb916dbc9" + assert json_data["peetprint_hash"] == "b8ce945a4d9a7a9b5b6132e3658fe033"