Skip to content

Commit

Permalink
Bump httpx
Browse files Browse the repository at this point in the history
Is has some internal changes that requires small tweak

params/json/etc. can't be set on the session anymore and must always be passed when doing get/post/etc.
As such they can't be extracted from the options.
A requests options also seem to only contain the session parameters and don't merge with get/post/etc.
  • Loading branch information
Earlopain committed Jul 22, 2024
1 parent 6d9499b commit 27ad400
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 12 deletions.
6 changes: 3 additions & 3 deletions Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,9 @@ GEM
railties (>= 6.1.0)
thor (>= 1.0.0)
hashdiff (1.1.0)
http-2-next (1.0.3)
httpx (1.2.6)
http-2-next (>= 1.0.3)
http-2 (1.0.0)
httpx (1.3.0)
http-2 (>= 1.0.0)
i18n (1.14.5)
concurrent-ruby (~> 1.0)
io-console (0.7.2)
Expand Down
7 changes: 4 additions & 3 deletions app/logical/scraper/httpx_plugin.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ module InstanceMethods
def request(method, uri, should_raise: true, **params)
response = scraper.enfore_rate_limit { super(method, uri, **params) }

request_options = response.instance_variable_get(:@request).options.to_hash
relevant_options = request_options.slice(:headers, :params, :json, :form, :body)
relevant_options.delete(:headers) if relevant_options[:headers].to_h.blank?
session_headers = response.instance_variable_get(:@request).options.headers
headers = session_headers.merge(params[:headers] || {}).to_h
relevant_options = params.slice(:params, :json, :form, :body)
relevant_options[:headers] = headers if headers.present?
scraper.log_response(response.uri, method, relevant_options, response.status, response.body.to_s)

response.raise_unless_ok if should_raise
Expand Down
11 changes: 5 additions & 6 deletions test/logical/scraper/httpx_plugin_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -72,24 +72,23 @@ class HttpxPluginTest < ActiveSupport::TestCase
end

test "it logs options that are set on the session" do
stub_request(:get, "https://example.com/test?foo=bar&baz=fiz")
stub_request(:get, "https://example.com/test?foo=bar")
@client.with(
origin: "https://example.com",
headers: { "x-a" => "a" },
params: { foo: "bar" },
).get("/test", headers: { "x-b" => "b" }, params: { baz: "fiz" })
).get("/test", headers: { "x-b" => "b" }, params: { foo: "bar" })

payload = @submission_file.log_events.sole.payload
assert_equal("https://example.com/test", payload["path"])
assert_equal({
"headers" => { "x-a" => "a", "x-b" => "b" },
"params" => { "foo" => "bar", "baz" => "fiz" },
"params" => { "foo" => "bar" },
}, payload["request_params"])
end

test "it logs json parameters" do
stub_request(:post, "https://example.com")
@client.with(json: { foo: "bar" }).post("https://example.com")
stub_request(:post, "https://example.com").with(body: { foo: "bar" }.to_json)
@client.post("https://example.com", json: { foo: "bar" })

payload = @submission_file.log_events.sole.payload
assert_equal("POST", payload["method"])
Expand Down

0 comments on commit 27ad400

Please sign in to comment.