Skip to content

Commit

Permalink
Utilities: Cleanup File Classes
Browse files Browse the repository at this point in the history
  • Loading branch information
@HTRamsey
HTRamsey committed Oct 25, 2024
1 parent 34cdb6b commit a92d071
Show file tree
Hide file tree
Showing 11 changed files with 253 additions and 184 deletions.
37 changes: 19 additions & 18 deletions src/Utilities/QGCCachedFileDownload.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,57 +7,58 @@
*
****************************************************************************/


#include "QGCCachedFileDownload.h"
#include "QGCFileDownload.h"

#include <QtCore/QDateTime>
#include <QtNetwork/QNetworkDiskCache>

QGCCachedFileDownload::QGCCachedFileDownload(QObject* parent, const QString& cacheDirectory)
: QObject(parent), _fileDownload(new QGCFileDownload(this)), _diskCache(new QNetworkDiskCache(this))
QGCCachedFileDownload::QGCCachedFileDownload(const QString &cacheDirectory, QObject *parent)
: QObject(parent)
, _fileDownload(new QGCFileDownload(this))
, _diskCache(new QNetworkDiskCache(this))
{
_diskCache->setCacheDirectory(cacheDirectory);
_fileDownload->setCache(_diskCache);

connect(_fileDownload, &QGCFileDownload::downloadProgress, this, &QGCCachedFileDownload::downloadProgress);
connect(_fileDownload, &QGCFileDownload::downloadComplete, this, &QGCCachedFileDownload::onDownloadCompleted);
(void) connect(_fileDownload, &QGCFileDownload::downloadProgress, this, &QGCCachedFileDownload::downloadProgress);
(void) connect(_fileDownload, &QGCFileDownload::downloadComplete, this, &QGCCachedFileDownload::_onDownloadCompleted);
}

bool QGCCachedFileDownload::download(const QString& url, int maxCacheAgeSec)
bool QGCCachedFileDownload::download(const QString &url, int maxCacheAgeSec)
{
_downloadFromNetwork = false;
// Check cache
QNetworkCacheMetaData metadata = _diskCache->metaData(url);
if (metadata.isValid() && metadata.attributes().contains(QNetworkRequest::Attribute::User)) {

const QNetworkCacheMetaData metadata = _diskCache->metaData(url);
if (metadata.isValid() && metadata.attributes().contains(QNetworkRequest::Attribute::User)) {
// We want the following behavior:
// - Use the cached file if not older than maxCacheAgeSec
// - Otherwise try to download the file, but still use the cached file if there's no connection

QDateTime creationTime = metadata.attributes().find(QNetworkRequest::Attribute::User)->toDateTime();
bool expired = creationTime.addSecs(maxCacheAgeSec) < QDateTime::currentDateTime();
const auto &metaDataAttributes = metadata.attributes();
const QDateTime creationTime = metaDataAttributes.find(QNetworkRequest::Attribute::User)->toDateTime();
const bool expired = creationTime.addSecs(maxCacheAgeSec) < QDateTime::currentDateTime();
if (expired) {
// Force network download, as Qt would still use the cache otherwise (w/o checking the remote)
auto attributes = QVector<QPair<QNetworkRequest::Attribute, QVariant>>{qMakePair(QNetworkRequest::CacheLoadControlAttribute, QVariant{QNetworkRequest::AlwaysNetwork})};
const auto attributes = QList<QPair<QNetworkRequest::Attribute, QVariant>>{qMakePair(QNetworkRequest::CacheLoadControlAttribute, QVariant{QNetworkRequest::AlwaysNetwork})};
_downloadFromNetwork = true;
return _fileDownload->download(url, attributes);
}

auto attributes = QVector<QPair<QNetworkRequest::Attribute, QVariant>>{qMakePair(QNetworkRequest::CacheLoadControlAttribute, QVariant{QNetworkRequest::PreferCache})};
const auto attributes = QList<QPair<QNetworkRequest::Attribute, QVariant>>{qMakePair(QNetworkRequest::CacheLoadControlAttribute, QVariant{QNetworkRequest::PreferCache})};
return _fileDownload->download(url, attributes);

} else {
return _fileDownload->download(url);
}

return _fileDownload->download(url);
}

void QGCCachedFileDownload::onDownloadCompleted(QString remoteFile, QString localFile, QString errorMsg)
void QGCCachedFileDownload::_onDownloadCompleted(const QString &remoteFile, const QString &localFile, const QString &errorMsg)
{
// Set cache creation time if not set already (the Qt docs mention there's a creation time, but I could not find any API)
QNetworkCacheMetaData metadata = _diskCache->metaData(remoteFile);
if (metadata.isValid() && !metadata.attributes().contains(QNetworkRequest::Attribute::User)) {
QNetworkCacheMetaData::AttributesMap attributes = metadata.attributes();
attributes.insert(QNetworkRequest::Attribute::User, QDateTime::currentDateTime());
(void) attributes.insert(QNetworkRequest::Attribute::User, QDateTime::currentDateTime());
metadata.setAttributes(attributes);
_diskCache->updateMetaData(metadata);
}
Expand Down
21 changes: 11 additions & 10 deletions src/Utilities/QGCCachedFileDownload.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,34 @@

#pragma once

#include <QtCore/QString>
#include <QtCore/QObject>
#include <QtCore/QString>

class QGCFileDownload;
class QNetworkDiskCache;

class QGCCachedFileDownload : public QObject
{
Q_OBJECT

public:
QGCCachedFileDownload(QObject* parent, const QString& cacheDirectory);
QGCCachedFileDownload(const QString &cacheDirectory, QObject *parent = nullptr);

/// Download the specified remote file.
/// @param url File to download
/// @param maxCacheAgeSec Maximum age of cached item in seconds
/// @return true: Asynchronous download has started, false: Download initialization failed
bool download(const QString& url, int maxCacheAgeSec);
bool download(const QString &url, int maxCacheAgeSec);

signals:
void downloadProgress(qint64 curr, qint64 total);
void downloadComplete(QString remoteFile, QString localFile, QString errorMsg);
void downloadComplete(const QString &remoteFile, const QString &localFile, const QString &errorMsg);

private:
void onDownloadCompleted(QString remoteFile, QString localFile, QString errorMsg);
private slots:
void _onDownloadCompleted(const QString &remoteFile, const QString &localFile, const QString &errorMsg);

QGCFileDownload* _fileDownload;
QNetworkDiskCache* _diskCache;
bool _downloadFromNetwork{false};
private:
QGCFileDownload* _fileDownload = nullptr;
QNetworkDiskCache* _diskCache = nullptr;
bool _downloadFromNetwork = false;
};
137 changes: 77 additions & 60 deletions src/Utilities/QGCFileDownload.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,101 +7,135 @@
*
****************************************************************************/


#include "QGCFileDownload.h"
#include "QGCLoggingCategory.h"

#include <QtCore/QFileInfo>
#include <QtCore/QStandardPaths>
#include <QtNetwork/QNetworkProxy>

QGCFileDownload::QGCFileDownload(QObject* parent)
: QNetworkAccessManager(parent)
QGC_LOGGING_CATEGORY(QGCFileDownloadLog, "qgc.utilities.qgcfiledownload");

QGCFileDownload::QGCFileDownload(QObject *parent)
: QObject(parent)
, _networkManager(new QNetworkAccessManager(this))
{
// qCDebug(QGCFileDownloadLog) << Q_FUNC_INFO << this;
}

QGCFileDownload::~QGCFileDownload()
{
// qCDebug(QGCFileDownloadLog) << Q_FUNC_INFO << this;
}

void QGCFileDownload::setCache(QAbstractNetworkCache *cache)
{
_networkManager->setCache(cache);
}

void QGCFileDownload::setIgnoreSSLErrorsIfNeeded(QNetworkReply &networkReply)
{
const bool sslLibraryBuildIs1x = ((QSslSocket::sslLibraryBuildVersionNumber() & 0xf0000000) == 0x10000000);
const bool sslLibraryIs3x = ((QSslSocket::sslLibraryVersionNumber() & 0xf0000000) == 0x30000000);
if (sslLibraryBuildIs1x && sslLibraryIs3x) {
qCWarning(QGCFileDownloadLog) << "Ignoring ssl certificates due to OpenSSL version mismatch";
QList<QSslError> errorsThatCanBeIgnored;
errorsThatCanBeIgnored << QSslError(QSslError::NoPeerCertificate);
networkReply.ignoreSslErrors(errorsThatCanBeIgnored);
}
}

bool QGCFileDownload::download(const QString& remoteFile, const QVector<QPair<QNetworkRequest::Attribute, QVariant>>& requestAttributes, bool redirect)
bool QGCFileDownload::download(const QString &remoteFile, const QList<QPair<QNetworkRequest::Attribute,QVariant>> &requestAttributes, bool redirect)
{
if (!redirect) {
_requestAttributes = requestAttributes;
_originalRemoteFile = remoteFile;
}

if (remoteFile.isEmpty()) {
qWarning() << "downloadFile empty";
qCWarning(QGCFileDownloadLog) << "downloadFile empty";
return false;
}


QUrl remoteUrl;
if (remoteFile.startsWith("http:") || remoteFile.startsWith("https:")) {
remoteUrl.setUrl(remoteFile);
} else {
remoteUrl = QUrl::fromLocalFile(remoteFile);
}

if (!remoteUrl.isValid()) {
qWarning() << "Remote URL is invalid" << remoteFile;
qCWarning(QGCFileDownloadLog) << "Remote URL is invalid" << remoteFile;
return false;
}

QNetworkRequest networkRequest(remoteUrl);

for (const auto& attribute : requestAttributes) {
QNetworkRequest networkRequest(remoteUrl);
for (const QPair<QNetworkRequest::Attribute,QVariant> &attribute : requestAttributes) {
networkRequest.setAttribute(attribute.first, attribute.second);
}

QNetworkProxy tProxy;
#if !defined(Q_OS_IOS) && !defined(Q_OS_ANDROID)
QNetworkProxy tProxy = _networkManager->proxy();
tProxy.setType(QNetworkProxy::DefaultProxy);
setProxy(tProxy);

QNetworkReply* networkReply = get(networkRequest);
_networkManager->setProxy(tProxy);
#endif

QNetworkReply *const networkReply = _networkManager->get(networkRequest);
if (!networkReply) {
qWarning() << "QNetworkAccessManager::get failed";
qCWarning(QGCFileDownloadLog) << "QNetworkAccessManager::get failed";
return false;
}

setIgnoreSSLErrorsIfNeeded(*networkReply);

connect(networkReply, &QNetworkReply::downloadProgress, this, &QGCFileDownload::downloadProgress);
connect(networkReply, &QNetworkReply::finished, this, &QGCFileDownload::_downloadFinished);
connect(networkReply, &QNetworkReply::errorOccurred, this, &QGCFileDownload::_downloadError);
(void) connect(networkReply, &QNetworkReply::downloadProgress, this, &QGCFileDownload::downloadProgress);
(void) connect(networkReply, &QNetworkReply::finished, this, &QGCFileDownload::_downloadFinished);
(void) connect(networkReply, &QNetworkReply::errorOccurred, this, &QGCFileDownload::_downloadError);

return true;
}

void QGCFileDownload::_downloadFinished(void)
void QGCFileDownload::_downloadFinished()
{
QNetworkReply* reply = qobject_cast<QNetworkReply*>(QObject::sender());
QNetworkReply *const reply = qobject_cast<QNetworkReply*>(QObject::sender());
if (!reply) {
return;
}
reply->deleteLater();

// When an error occurs or the user cancels the download, we still end up here. So bail out in
// those cases.
if (reply->error() != QNetworkReply::NoError) {
reply->deleteLater();
return;
}

// Check for redirection
if (!reply->isOpen()) {
return;
}

const int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if ((statusCode < HTTP_Response::SUCCESS_OK) || (statusCode >= HTTP_Response::REDIRECTION_MULTIPLE_CHOICES)) {
return;
}

QVariant redirectionTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute);
if (!redirectionTarget.isNull()) {
QUrl redirectUrl = reply->url().resolved(redirectionTarget.toUrl());
download(redirectUrl.toString(), _requestAttributes, true /* redirect */);
reply->deleteLater();
const QUrl redirectUrl = reply->url().resolved(redirectionTarget.toUrl());
(void) download(redirectUrl.toString(), _requestAttributes, true);
return;
}

// Split out filename from path
QString remoteFileName = QFileInfo(reply->url().toString()).fileName();
if (remoteFileName.isEmpty()) {
qWarning() << "Unabled to parse filename from remote url" << reply->url().toString();
qCWarning(QGCFileDownloadLog) << "Unabled to parse filename from remote url" << reply->url().toString();
remoteFileName = "DownloadedFile";
}

// Strip out http parameters from remote filename
int parameterIndex = remoteFileName.indexOf("?");
const int parameterIndex = remoteFileName.indexOf("?");
if (parameterIndex != -1) {
remoteFileName = remoteFileName.left(parameterIndex);
remoteFileName = remoteFileName.left(parameterIndex);
}

// Determine location to download file to
QString downloadFilename = QStandardPaths::writableLocation(QStandardPaths::TempLocation);
if (downloadFilename.isEmpty()) {
downloadFilename = QStandardPaths::writableLocation(QStandardPaths::DownloadLocation);
Expand All @@ -110,13 +144,13 @@ void QGCFileDownload::_downloadFinished(void)
return;
}
}
downloadFilename += "/" + remoteFileName;
downloadFilename += "/" + remoteFileName;

if (!downloadFilename.isEmpty()) {
// Store downloaded file in download location
QFile file(downloadFilename);
if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
emit downloadComplete(_originalRemoteFile, downloadFilename, tr("Could not save downloaded file to %1. Error: %2").arg(downloadFilename).arg(file.errorString()));
emit downloadComplete(_originalRemoteFile, downloadFilename, tr("Could not save downloaded file to %1. Error: %2").arg(downloadFilename, file.errorString()));
return;
}

Expand All @@ -125,44 +159,27 @@ void QGCFileDownload::_downloadFinished(void)

emit downloadComplete(_originalRemoteFile, downloadFilename, QString());
} else {
QString errorMsg = "Internal error";
qWarning() << errorMsg;
const QString errorMsg = "Internal error";
qCWarning(QGCFileDownloadLog) << errorMsg;
emit downloadComplete(_originalRemoteFile, downloadFilename, errorMsg);
}

reply->deleteLater();
}

/// @brief Called when an error occurs during download
void QGCFileDownload::_downloadError(QNetworkReply::NetworkError code)
{
QString errorMsg;

if (code == QNetworkReply::OperationCanceledError) {
errorMsg = tr("Download cancelled");

} else if (code == QNetworkReply::ContentNotFoundError) {
switch (code) {
case QNetworkReply::OperationCanceledError:
errorMsg = tr("Download cancelled");
break;
case QNetworkReply::ContentNotFoundError:
errorMsg = tr("Error: File Not Found");

} else {
break;
default:
errorMsg = tr("Error during download. Error: %1").arg(code);
break;
}

emit downloadComplete(_originalRemoteFile, QString(), errorMsg);
}

void QGCFileDownload::setIgnoreSSLErrorsIfNeeded(QNetworkReply& networkReply)
{
// Some systems (like Ubuntu 22.04) only ship with OpenSSL 3.x, however Qt 5.15.2 links against OpenSSL 1.x.
// This results in unresolved symbols for EVP_PKEY_base_id and SSL_get_peer_certificate.
// To still get a connection we have to ignore certificate verification (connection is still encrypted but open to MITM attacks)
// See https://bugreports.qt.io/browse/QTBUG-115146
const bool sslLibraryBuildIs1x = (QSslSocket::sslLibraryBuildVersionNumber() & 0xf0000000) == 0x10000000;
const bool sslLibraryIs3x = (QSslSocket::sslLibraryVersionNumber() & 0xf0000000) == 0x30000000;
if (sslLibraryBuildIs1x && sslLibraryIs3x) {
qWarning() << "Ignoring ssl certificates due to OpenSSL version mismatch";
QList<QSslError> errorsThatCanBeIgnored;
errorsThatCanBeIgnored << QSslError(QSslError::NoPeerCertificate);
networkReply.ignoreSslErrors(errorsThatCanBeIgnored);
}
}
Loading

0 comments on commit a92d071

Please sign in to comment.