Skip to content
This repository has been archived by the owner on Dec 24, 2022. It is now read-only.

Commit

Permalink
Merge remote-tracking branch 'origin/master'
Browse files Browse the repository at this point in the history
weblate committed May 28, 2016

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
2 parents 5ac5fe8 + eb75c1d commit 2df235e
Showing 3 changed files with 49 additions and 353 deletions.
81 changes: 26 additions & 55 deletions mobile/src/main/java/net/etuldan/sparss/service/FetcherService.java
Original file line number Diff line number Diff line change
@@ -60,7 +60,8 @@
import android.net.Uri;
import android.os.Handler;
import android.os.SystemClock;
import android.util.Log;
import android.text.Html;
import android.text.TextUtils;
import android.util.Xml;
import android.widget.Toast;

@@ -78,19 +79,17 @@
import net.etuldan.sparss.utils.NetworkUtils;
import net.etuldan.sparss.utils.PrefUtils;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Date;
import java.util.concurrent.Callable;
@@ -104,7 +103,6 @@
import java.util.regex.Pattern;

public class FetcherService extends IntentService {
private static final String TAG = "FetcherService";

public static final String ACTION_REFRESH_FEEDS = "net.etuldan.sparss.REFRESH";
public static final String ACTION_MOBILIZE_FEEDS = "net.etuldan.sparss.MOBILIZE_FEEDS";
@@ -190,31 +188,29 @@ public void run() {
}
});
}
Log.d(TAG, "onHandleIntent: "+intent.getAction()+" aborted due to connectivity problem");
return;
}

boolean skipFetch = isFromAutoRefresh && PrefUtils.getBoolean(PrefUtils.REFRESH_WIFI_ONLY, false)
&& networkInfo.getType() != ConnectivityManager.TYPE_WIFI;
// We need to skip the fetching process, so we quit
if (skipFetch) {
Log.d(TAG, "onHandleIntent: abort intent action: " + intent.getAction() + " due to connectivity settings");
return;
}

Log.d(TAG, "onHandleIntent: intent action: " + intent.getAction());
PrefUtils.putBoolean(PrefUtils.IS_REFRESHING, true);
if (ACTION_MOBILIZE_FEEDS.equals(intent.getAction())) {
mobilizeAllEntries();
downloadAllImages();
} else if (ACTION_DOWNLOAD_IMAGES.equals(intent.getAction())) {
downloadAllImages();
} else { // == Constants.ACTION_REFRESH_FEEDS
PrefUtils.putBoolean(PrefUtils.IS_REFRESHING, true);

if (isFromAutoRefresh) {
PrefUtils.putLong(PrefUtils.LAST_SCHEDULED_REFRESH, SystemClock.elapsedRealtime());
}

long keepTime = Long.parseLong(PrefUtils.getString(PrefUtils.KEEP_TIME, "4")) * 86400000L;
long keepTime = Long.parseLong(PrefUtils.getString(PrefUtils.KEEP_TIME, "4")) * 86400000l;
long keepDateBorderTime = keepTime > 0 ? System.currentTimeMillis() - keepTime : 0;

deleteOldEntries(keepDateBorderTime);
@@ -272,8 +268,9 @@ public void run() {

mobilizeAllEntries();
downloadAllImages();

PrefUtils.putBoolean(PrefUtils.IS_REFRESHING, false);
}
PrefUtils.putBoolean(PrefUtils.IS_REFRESHING, false);
}

private void mobilizeAllEntries() {
@@ -298,10 +295,7 @@ private void mobilizeAllEntries() {

if (entryCursor.moveToFirst()) {
if (entryCursor.isNull(entryCursor.getColumnIndex(EntryColumns.MOBILIZED_HTML))) { // If we didn't already mobilized it
Log.d(TAG, "mobilizeAllEntries: mobilizing entry " + entryId);

int linkPos = entryCursor.getColumnIndex(EntryColumns.LINK);
int titlePos = entryCursor.getColumnIndex(EntryColumns.TITLE);
int abstractHtmlPos = entryCursor.getColumnIndex(EntryColumns.ABSTRACT);
int feedIdPos = entryCursor.getColumnIndex(EntryColumns.FEED_ID);
HttpURLConnection connection = null;
@@ -321,32 +315,22 @@ private void mobilizeAllEntries() {
final String httpAuthPassValue = cursorFeed.getString(httpAuthPasswordPosition);
cursorFeed.close();

String fullSummary = entryCursor.getString(abstractHtmlPos);

String mobilizedHtml;
if(fullSummary.length() > 1000) {
//if summary is long, it is most probably full text. use it!
mobilizedHtml = fullSummary;
} else {
// Try to find a text indicator for better content extraction
Document doc = Jsoup.parse(fullSummary);
String contentIndicator = doc.text().substring(0, Math.min(doc.text().length(), 100));

String titleIndicator = entryCursor.getString(titlePos);
doc = Jsoup.parse(titleIndicator);
titleIndicator = doc.text();

// titleIndicator = Html.fromHtml(titleIndicator).toString();
// titleIndicator = titleIndicator.replaceAll("[\\s\\u00A0]+"," "); //normalize, all whitespaces (incl char(160)) -> single space
connection = NetworkUtils.setupConnection(link, cookieName, cookieValue, httpAuthLoginValue, httpAuthPassValue);

mobilizedHtml = ArticleTextExtractor.extractContent(HtmlUtils.decompressStream(connection.getInputStream()), contentIndicator, titleIndicator);
if(mobilizedHtml != null) {
mobilizedHtml = HtmlUtils.improveHtmlContent(mobilizedHtml, NetworkUtils.getBaseUrl(connection.getURL().toURI().toString()));
// Try to find a text indicator for better content extraction
String contentIndicator = null;
String text = entryCursor.getString(abstractHtmlPos);
if (!TextUtils.isEmpty(text)) {
text = Html.fromHtml(text).toString();
if (text.length() > 60) {
contentIndicator = text.substring(20, 40);
}
}

connection = NetworkUtils.setupConnection(link,cookieName, cookieValue,httpAuthLoginValue, httpAuthPassValue);

String mobilizedHtml = ArticleTextExtractor.extractContent(connection.getInputStream(), contentIndicator);

if (mobilizedHtml != null) {
mobilizedHtml = HtmlUtils.improveHtmlContent(mobilizedHtml, NetworkUtils.getBaseUrl(link));
ContentValues values = new ContentValues();
values.put(EntryColumns.MOBILIZED_HTML, mobilizedHtml);

@@ -375,14 +359,12 @@ private void mobilizeAllEntries() {
}
}
} catch (Throwable ignored) {
Log.e(TAG, "Exception: " + ignored.getMessage(), ignored);
} finally {
if (connection != null) {
connection.disconnect();
}
}
} else { // We already mobilized it
Log.d(TAG, "mobilizeAllEntries: entry " + entryId + "@" + entryUri + " already mobilized");
success = true;
operations.add(ContentProviderOperation.newDelete(TaskColumns.CONTENT_URI(taskId)).build());
}
@@ -406,7 +388,6 @@ private void mobilizeAllEntries() {
try {
cr.applyBatch(FeedData.AUTHORITY, operations);
} catch (Throwable ignored) {
Log.e(TAG, "Exception", ignored);
}
}
}
@@ -440,7 +421,6 @@ private void downloadAllImages() {
values.put(TaskColumns.NUMBER_ATTEMPT, nbAttempt + 1);
operations.add(ContentProviderOperation.newUpdate(TaskColumns.CONTENT_URI(taskId)).withValues(values).build());
}
Log.e(TAG, "downloadAllImages: Exception", e);
}
}

@@ -450,7 +430,6 @@ private void downloadAllImages() {
try {
cr.applyBatch(FeedData.AUTHORITY, operations);
} catch (Throwable ignored) {
Log.e(TAG, "Exception", ignored);
}
}
}
@@ -499,7 +478,6 @@ public Integer call() {
try {
result = refreshFeed(feedId, keepDateBorderTime);
} catch (Exception ignored) {
Log.e(TAG, "Exception", ignored);
}
return result;
}
@@ -513,7 +491,6 @@ public Integer call() {
Future<Integer> f = completionService.take();
globalResult += f.get();
} catch (Exception ignored) {
Log.e(TAG, "Exception", ignored);
}
}

@@ -557,7 +534,7 @@ private int refreshFeed(String feedId, long keepDateBorderTime) {

if (fetchMode == 0) {
if (contentType != null && contentType.startsWith(CONTENT_TYPE_TEXT_HTML)) {
BufferedReader reader = new BufferedReader(new InputStreamReader(HtmlUtils.decompressStream(connection.getInputStream())));
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));

String line;
int posStart = -1;
@@ -616,15 +593,14 @@ private int refreshFeed(String feedId, long keepDateBorderTime) {
Xml.findEncodingByName(index2 > -1 ? contentType.substring(index + 8, index2) : contentType.substring(index + 8));
fetchMode = FETCHMODE_DIRECT;
} catch (UnsupportedEncodingException ignored) {
Log.e(TAG, "Exception", ignored);
fetchMode = FETCHMODE_REENCODE;
}
} else {
fetchMode = FETCHMODE_REENCODE;
}

} else {
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(HtmlUtils.decompressStream(connection.getInputStream())));
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(connection.getInputStream()));

char[] chars = new char[20];

@@ -642,7 +618,6 @@ private int refreshFeed(String feedId, long keepDateBorderTime) {
Xml.findEncodingByName(xmlDescription.substring(start + 10, xmlDescription.indexOf('"', start + 11)));
fetchMode = FETCHMODE_DIRECT;
} catch (UnsupportedEncodingException ignored) {
Log.e(TAG, "Exception", ignored);
fetchMode = FETCHMODE_REENCODE;
}
} else {
@@ -663,19 +638,19 @@ private int refreshFeed(String feedId, long keepDateBorderTime) {
int index = contentType.indexOf(CHARSET);
int index2 = contentType.indexOf(';', index);

InputStream inputStream = HtmlUtils.decompressStream(connection.getInputStream());
InputStream inputStream = connection.getInputStream();
Xml.parse(inputStream,
Xml.findEncodingByName(index2 > -1 ? contentType.substring(index + 8, index2) : contentType.substring(index + 8)),
handler);
} else {
InputStreamReader reader = new InputStreamReader(HtmlUtils.decompressStream(connection.getInputStream()));
InputStreamReader reader = new InputStreamReader(connection.getInputStream());
Xml.parse(reader, handler);
}
break;
}
case FETCHMODE_REENCODE: {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
InputStream inputStream = HtmlUtils.decompressStream(connection.getInputStream());
InputStream inputStream = connection.getInputStream();

byte[] byteBuffer = new byte[4096];

@@ -705,7 +680,6 @@ private int refreshFeed(String feedId, long keepDateBorderTime) {
index + 8, index2) : contentType.substring(index + 8)));
Xml.parse(reader, handler);
} catch (Exception ignored) {
Log.e(TAG, "Exception", ignored);
}
} else {
StringReader reader = new StringReader(xmlText);
@@ -728,7 +702,6 @@ private int refreshFeed(String feedId, long keepDateBorderTime) {
values.put(FeedColumns.ERROR, getString(R.string.error_feed_error));
cr.update(FeedColumns.CONTENT_URI(id), values, null, null);
}
Log.e(TAG, "refreshFeed: FileNotFoundException: ", e);
} catch (Throwable e) {
if (handler == null || (!handler.isDone() && !handler.isCancelled())) {
ContentValues values = new ContentValues();
@@ -739,7 +712,6 @@ private int refreshFeed(String feedId, long keepDateBorderTime) {
values.put(FeedColumns.ERROR, e.getMessage() != null ? e.getMessage() : getString(R.string.error_feed_process));
cr.update(FeedColumns.CONTENT_URI(id), values, null, null);
}
Log.e(TAG, "refreshFeed: Exception: ", e);
} finally {

/* check and optionally find favicon */
@@ -753,7 +725,6 @@ private int refreshFeed(String feedId, long keepDateBorderTime) {
}
}
} catch (Throwable ignored) {
Log.d(TAG, "Exception favicon could not be retrieved.");
}

if (connection != null) {
Loading

0 comments on commit 2df235e

Please sign in to comment.