Return the response body when a HTTP error occurs during scraping

s9e · Apr 25, 2024 · 742e3da · 742e3da
1 parent 05b9223
commit 742e3da
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 14 deletions.
diff --git a/docs/testdox.txt b/docs/testdox.txt
@@ -7947,7 +7947,7 @@ Cached (s9e\TextFormatter\Tests\Utils\Http\Clients\Cached)
  [x] post() sends the request body if set
  [x] post() sends no request body if not set
  [x] post() automatically sets Content-Length if a request body is set
- [x] get() returns FALSE on error
+ [x] get() returns the response error
  [x] get() returns headers if returnHeaders is true
 
 Curl (s9e\TextFormatter\Tests\Utils\Http\Clients\Curl)
@@ -7957,7 +7957,7 @@ Curl (s9e\TextFormatter\Tests\Utils\Http\Clients\Curl)
  [x] post() sends the request body if set
  [x] post() sends no request body if not set
  [x] post() automatically sets Content-Length if a request body is set
- [x] get() returns FALSE on error
+ [ ] get() returns the response error
  [x] get() returns headers if returnHeaders is true
 
 Native (s9e\TextFormatter\Tests\Utils\Http\Clients\Native)
@@ -7968,7 +7968,7 @@ Native (s9e\TextFormatter\Tests\Utils\Http\Clients\Native)
  [x] post() sends the request body if set
  [x] post() sends no request body if not set
  [x] post() automatically sets Content-Length if a request body is set
- [x] get() returns FALSE on error
+ [x] get() returns the response error
  [x] get() returns headers if returnHeaders is true
 
 Http (s9e\TextFormatter\Tests\Utils\Http)

diff --git a/src/Utils/Http/Clients/Curl.php b/src/Utils/Http/Clients/Curl.php
@@ -24,10 +24,11 @@ public function get($url, array $options = [])
 		$options += ['headers' => []];
 
 		$handle = $this->getHandle();
-		curl_setopt($handle, CURLOPT_HEADER,     !empty($options['returnHeaders']));
-		curl_setopt($handle, CURLOPT_HTTPGET,    true);
-		curl_setopt($handle, CURLOPT_HTTPHEADER, $options['headers']);
-		curl_setopt($handle, CURLOPT_URL,        $url);
+		curl_setopt($handle, CURLOPT_FAILONERROR, false);
+		curl_setopt($handle, CURLOPT_HEADER,      !empty($options['returnHeaders']));
+		curl_setopt($handle, CURLOPT_HTTPGET,     true);
+		curl_setopt($handle, CURLOPT_HTTPHEADER,  $options['headers']);
+		curl_setopt($handle, CURLOPT_URL,         $url);
 
 		return curl_exec($handle);
 	}

diff --git a/src/Utils/Http/Clients/Native.php b/src/Utils/Http/Clients/Native.php
@@ -53,10 +53,11 @@ protected function createContext($method, array $options, $body)
 		$contextOptions = [
 			'ssl'  => ['verify_peer' => $this->sslVerifyPeer],
 			'http' => [
-				'method'  => $method,
-				'timeout' => $this->timeout,
-				'header'  => $this->generateHeaders($options, $body),
-				'content' => $body
+				'content'       => $body,
+				'header'        => $this->generateHeaders($options, $body),
+				'ignore_errors' => true,
+				'method'        => $method,
+				'timeout'       => $this->timeout
 			]
 		];
 

diff --git a/tests/Utils/Http/Clients/AbstractTestClass.php b/tests/Utils/Http/Clients/AbstractTestClass.php
@@ -105,11 +105,14 @@ public function testPostContentLength()
 	}
 
 	/**
-	* @testdox get() returns FALSE on error
+	* @testdox get() returns the response error
 	*/
-	public function testReturnsFalse()
+	public function testReturnsResponseOnError()
 	{
-		$this->assertFalse($this->getInstance()->get(str_replace('reflect.php', '404', $this->url)));
+		$this->assertStringContainsString(
+			'404 Not Found',
+			$this->getInstance()->get(str_replace('reflect.php', '404', $this->url))
+		);
 	}
 
 	/**