Feature/sync unix symlinks as links

.gitignore

@@ -51,6 +51,9 @@ bld/
 [Tt]est[Rr]esult*/
 [Bb]uild[Ll]og.*
 
+# clangd cache directory
+.cache/
+
 #NUNIT
 *.VisualState.xml
 TestResult.xml

CMakeLists.txt

@@ -7,6 +7,8 @@ if (CLANG_TIDY_EXE)
     set(CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY_EXE} -checks=-*,modernize-use-auto,modernize-use-using,modernize-use-nodiscard,modernize-use-nullptr,modernize-use-override,cppcoreguidelines-pro-type-static-cast-downcast,modernize-use-default-member-init,cppcoreguidelines-pro-type-member-init,cppcoreguidelines-init-variables)
 endif()
 
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
 project(client)
 
 include(FeatureSummary)

src/common/checksumcalculator.cpp

@@ -12,11 +12,14 @@
  * for more details.
  */
 #include "checksumcalculator.h"
+#include "filesystembase.h"
 
 #include <zlib.h>
 
+#include <QBuffer>
 #include <QCryptographicHash>
 #include <QFile>
+#include <QFileInfo>
 #include <QLoggingCategory>
 
 namespace
@@ -48,7 +51,7 @@ static QCryptographicHash::Algorithm algorithmTypeToQCryptoHashAlgorithm(Checksu
 }
 
 ChecksumCalculator::ChecksumCalculator(const QString &filePath, const QByteArray &checksumTypeName)
-    : _device(new QFile(filePath))
+    : _device(openFile(filePath))
 {
     if (checksumTypeName == checkSumMD5C) {
         _algorithmType = AlgorithmType::MD5;
@@ -140,6 +143,18 @@ QByteArray ChecksumCalculator::calculate()
     return result;
 }
 
+QScopedPointer<QIODevice> ChecksumCalculator::openFile(const QString &filePath)
+{
+    if (QFileInfo(filePath).isSymLink()) {
+        auto symlinkContent = FileSystem::readlink(filePath);
+        QScopedPointer<QBuffer> symlinkDevice(new QBuffer());
+        symlinkDevice->setData(symlinkContent);
+        return QScopedPointer<QIODevice>(symlinkDevice.take());
+    } else {
+        return QScopedPointer<QIODevice>(new QFile(filePath));
+    }
+}
+
 void ChecksumCalculator::initChecksumAlgorithm()
 {
     if (_algorithmType == AlgorithmType::Undefined) {

src/common/checksumcalculator.h

@@ -46,6 +46,7 @@ class OCSYNC_EXPORT ChecksumCalculator
     [[nodiscard]] QByteArray calculate();
 
 private:
+    static QScopedPointer<QIODevice> openFile(const QString &filePath);
     void initChecksumAlgorithm();
     bool addChunk(const QByteArray &chunk, const qint64 size);
     QScopedPointer<QIODevice> _device;

src/common/filesystembase.cpp

@@ -20,6 +20,8 @@
 #include "utility.h"
 #include "common/asserts.h"
 
+#include <filesystem>
+
 #include <QDateTime>
 #include <QDir>
 #include <QUrl>
@@ -127,6 +129,15 @@ bool FileSystem::setFileReadOnlyWeak(const QString &filename, bool readonly)
     return true;
 }
 
+QByteArray FileSystem::readlink(const QString &filename)
+{
+    if (!QFileInfo(filename).isSymLink()) {
+        return QByteArray();
+    }
+    auto symlinkContent = std::filesystem::read_symlink(filename.toStdString()).string();
+    return QByteArray(symlinkContent.data());
+}
+
 bool FileSystem::rename(const QString &originFileName,
     const QString &destinationFileName,
     QString *errorString)
@@ -312,13 +323,13 @@ bool FileSystem::fileExists(const QString &filename, const QFileInfo &fileInfo)
         return fileExistsWin(filename);
     }
 #endif
-    bool re = fileInfo.exists();
+    bool re = fileInfo.exists() || fileInfo.isSymLink();
     // if the filename is different from the filename in fileInfo, the fileInfo is
     // not valid. There needs to be one initialised here. Otherwise the incoming
     // fileInfo is re-used.
     if (fileInfo.filePath() != filename) {
         QFileInfo myFI(filename);
-        re = myFI.exists();
+        re = myFI.exists() || myFI.isSymLink();
     }
     return re;
 }

src/common/filesystembase.h

@@ -84,6 +84,16 @@ namespace FileSystem {
      */
     bool OCSYNC_EXPORT fileExists(const QString &filename, const QFileInfo & = QFileInfo());
 
+    /**
+     * @brief Return raw content of symlink at given path.
+     *
+     * If the file is not a symlink or does not exist, the returned string will be empty.
+     * In Qt6.6+, QFileInfo::readSymLink() can be used instead.
+     * QFileInfo::symLinkTarget() can *not* be used because it transforms the target to an
+     * absolute path which might break relative symlinks for cross-device synchronization.
+     */
+    QByteArray OCSYNC_EXPORT readlink(const QString &filename);
+
     /**
      * @brief Rename the file \a originFileName to \a destinationFileName.
      *

src/common/syncjournalfilerecord.h

@@ -65,6 +65,7 @@ class OCSYNC_EXPORT SyncJournalFileRecord
     [[nodiscard]] QDateTime modDateTime() const { return Utility::qDateTimeFromTime_t(_modtime); }
 
     [[nodiscard]] bool isDirectory() const { return _type == ItemTypeDirectory; }
+    [[nodiscard]] bool isSymLink() const { return _type == ItemTypeSoftLink; }
     [[nodiscard]] bool isFile() const { return _type == ItemTypeFile || _type == ItemTypeVirtualFileDehydration; }
     [[nodiscard]] bool isVirtualFile() const { return _type == ItemTypeVirtualFile || _type == ItemTypeVirtualFileDownload; }
     [[nodiscard]] QString path() const { return QString::fromUtf8(_path); }

src/csync/std/c_time.cpp

@@ -25,8 +25,8 @@
 #include <QFile>
 
 #ifdef HAVE_UTIMES
-int c_utimes(const QString &uri, const struct timeval *times) {
-    int ret = utimes(QFile::encodeName(uri).constData(), times);
+int c_utimes(const QString &uri, const struct timespec *times) {
+    int ret = utimensat(AT_FDCWD, QFile::encodeName(uri).constData(), times, AT_SYMLINK_NOFOLLOW);
     return ret;
 }
 #else // HAVE_UTIMES
@@ -39,15 +39,15 @@ int c_utimes(const QString &uri, const struct timeval *times) {
 #define CSYNC_SECONDS_SINCE_1601 11644473600LL
 #define CSYNC_USEC_IN_SEC            1000000LL
 //after Microsoft KB167296
-static void UnixTimevalToFileTime(struct timeval t, LPFILETIME pft)
+static void UnixTimevalToFileTime(struct timespec t, LPFILETIME pft)
 {
     LONGLONG ll = 0;
-    ll = Int32x32To64(t.tv_sec, CSYNC_USEC_IN_SEC*10) + t.tv_usec*10 + CSYNC_SECONDS_SINCE_1601*CSYNC_USEC_IN_SEC*10;
+    ll = Int32x32To64(t.tv_sec, CSYNC_USEC_IN_SEC*10) + t.tv_nsec/100 + CSYNC_SECONDS_SINCE_1601*CSYNC_USEC_IN_SEC*10;
     pft->dwLowDateTime = (DWORD)ll;
     pft->dwHighDateTime = ll >> 32;
 }
 
-int c_utimes(const QString &uri, const struct timeval *times) {
+int c_utimes(const QString &uri, const struct timespec *times) {
     FILETIME LastAccessTime;
     FILETIME LastModificationTime;
     HANDLE hFile = nullptr;

src/csync/std/c_time.h

@@ -31,7 +31,7 @@
 #include <sys/time.h>
 #endif
 
-OCSYNC_EXPORT int c_utimes(const QString &uri, const struct timeval *times);
+OCSYNC_EXPORT int c_utimes(const QString &uri, const struct timespec *times);
 
 
 #endif /* _C_TIME_H */

src/gui/conflictdialog.cpp

@@ -16,6 +16,7 @@
 #include "ui_conflictdialog.h"
 
 #include "conflictsolver.h"
+#include "filesystem.h"
 
 #include <QDateTime>
 #include <QDebug>
@@ -132,9 +133,10 @@ void ConflictDialog::updateWidgets()
         const auto fileUrl = QUrl::fromLocalFile(filename).toString();
         linkLabel->setText(QStringLiteral("<a href='%1'>%2</a>").arg(fileUrl).arg(linkText));
 
-        const auto info = QFileInfo(filename);
-        mtimeLabel->setText(info.lastModified().toString());
-        sizeLabel->setText(locale().formattedDataSize(info.size()));
+        const auto lastModified = QDateTime::fromTime_t(FileSystem::getModTime(filename));
+        const auto fileSize = FileSystem::getSize(filename);
+        mtimeLabel->setText(lastModified.toString());
+        sizeLabel->setText(locale().formattedDataSize(fileSize));
 
         const auto mime = mimeDb.mimeTypeForFile(filename);
         if (QIcon::hasThemeIcon(mime.iconName())) {

src/gui/folder.cpp

@@ -1096,6 +1096,7 @@ SyncOptions Folder::initializeSyncOptions() const
     auto newFolderLimit = cfgFile.newBigFolderSizeLimit();
     opt._newBigFolderSizeLimit = newFolderLimit.first ? newFolderLimit.second * 1000LL * 1000LL : -1; // convert from MB to B
     opt._confirmExternalStorage = cfgFile.confirmExternalStorage();
+    opt._synchronizeSymlinks = cfgFile.synchronizeSymlinks();
     opt._moveFilesToTrash = cfgFile.moveToTrash();
     opt._vfs = _vfs;
     opt._parallelNetworkJobs = _accountState->account()->isHttp2Supported() ? 20 : 6;

src/gui/generalsettings.cpp

@@ -191,6 +191,7 @@ GeneralSettings::GeneralSettings(QWidget *parent)
     connect(_ui->existingFolderLimitCheckBox, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
     connect(_ui->stopExistingFolderNowBigSyncCheckBox, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
     connect(_ui->newExternalStorage, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
+    connect(_ui->synchronizeSymlinks, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
     connect(_ui->moveFilesToTrashCheckBox, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
 
 #ifndef WITH_CRASHREPORTER
@@ -259,6 +260,7 @@ void GeneralSettings::loadMiscSettings()
     _ui->showInExplorerNavigationPaneCheckBox->setChecked(cfgFile.showInExplorerNavigationPane());
     _ui->crashreporterCheckBox->setChecked(cfgFile.crashReporter());
     _ui->newExternalStorage->setChecked(cfgFile.confirmExternalStorage());
+    _ui->synchronizeSymlinks->setChecked(cfgFile.synchronizeSymlinks());
     _ui->monoIconsCheckBox->setChecked(cfgFile.monoIcons());
     _ui->moveFilesToTrashCheckBox->setChecked(cfgFile.moveToTrash());
 
@@ -270,6 +272,7 @@ void GeneralSettings::loadMiscSettings()
     _ui->stopExistingFolderNowBigSyncCheckBox->setEnabled(_ui->existingFolderLimitCheckBox->isChecked());
     _ui->stopExistingFolderNowBigSyncCheckBox->setChecked(_ui->existingFolderLimitCheckBox->isChecked() && cfgFile.stopSyncingExistingFoldersOverLimit());
     _ui->newExternalStorage->setChecked(cfgFile.confirmExternalStorage());
+    _ui->synchronizeSymlinks->setChecked(cfgFile.synchronizeSymlinks());
     _ui->monoIconsCheckBox->setChecked(cfgFile.monoIcons());
 }
 
@@ -447,6 +450,7 @@ void GeneralSettings::saveMiscSettings()
     cfgFile.setMoveToTrash(_ui->moveFilesToTrashCheckBox->isChecked());
     cfgFile.setNewBigFolderSizeLimit(newFolderLimitEnabled, _ui->newFolderLimitSpinBox->value());
     cfgFile.setConfirmExternalStorage(_ui->newExternalStorage->isChecked());
+    cfgFile.setSynchronizeSymlinks(_ui->synchronizeSymlinks->isChecked());
     cfgFile.setNotifyExistingFoldersOverLimit(existingFolderLimitEnabled);
     cfgFile.setStopSyncingExistingFoldersOverLimit(stopSyncingExistingFoldersOverLimit);
 

src/gui/generalsettings.ui

@@ -139,6 +139,17 @@
         </item>
        </layout>
       </item>
+      <item>
+       <layout class="QHBoxLayout" name="horizontalLayout_11">
+        <item>
+         <widget class="QCheckBox" name="synchronizeSymlinks">
+          <property name="text">
+           <string>[experimental] Synchronize symlinks</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
       <item>
        <layout class="QHBoxLayout" name="horizontalLayout">
         <item>

src/gui/tray/activitylistmodel.cpp

@@ -613,7 +613,7 @@ void ActivityListModel::addIgnoredFileToList(const Activity &newActivity)
     bool duplicate = false;
     if (_listOfIgnoredFiles.size() == 0) {
         _notificationIgnoredFiles = newActivity;
-        _notificationIgnoredFiles._subject = tr("Files from the ignore list as well as symbolic links are not synced.");
+        _notificationIgnoredFiles._subject = tr("Files from the ignore list are not synced.");
         addEntriesToActivityList({_notificationIgnoredFiles});
         _listOfIgnoredFiles.append(newActivity);
         return;

src/libsync/CMakeLists.txt

@@ -85,6 +85,8 @@ set(libsync_SRCS
     propagateuploadencrypted.cpp
     propagatedownloadencrypted.h
     propagatedownloadencrypted.cpp
+    symlinkuploaddevice.h
+    symlinkuploaddevice.cpp
     syncengine.h
     syncengine.cpp
     syncfileitem.h

src/libsync/bulkpropagatorjob.cpp

@@ -16,6 +16,7 @@
 
 #include "putmultifilejob.h"
 #include "owncloudpropagator_p.h"
+#include "symlinkuploaddevice.h"
 #include "syncfileitem.h"
 #include "syncengine.h"
 #include "propagateupload.h"
@@ -203,10 +204,18 @@ void BulkPropagatorJob::triggerUpload()
     int timeout = 0;
     for(auto &singleFile : _filesToUpload) {
         // job takes ownership of device via a QScopedPointer. Job deletes itself when finishing
-        auto device = std::make_unique<UploadDevice>(singleFile._localPath,
+        std::unique_ptr<UploadDevice> device;
+        if (singleFile._item->isSymLink()) {
+            device = std::make_unique<SymLinkUploadDevice>(singleFile._localPath,
+                                                            0,
+                                                            std::numeric_limits<qint64>::max(),
+                                                            &propagator()->_bandwidthManager);
+        } else {
+            device = std::make_unique<UploadDevice>(singleFile._localPath,
                                                      0,
                                                      singleFile._fileSize,
                                                      &propagator()->_bandwidthManager);
+        }
 
         if (!device->open(QIODevice::ReadOnly)) {
             qCWarning(lcBulkPropagatorJob) << "Could not prepare upload device: " << device->errorString();
@@ -224,6 +233,7 @@ void BulkPropagatorJob::triggerUpload()
         }
 
         singleFile._headers["X-File-Path"] = singleFile._remotePath.toUtf8();
+        singleFile._headers["OC-File-Type"] = QString::number(singleFile._item->_type).toUtf8();
         uploadParametersData.push_back({std::move(device), singleFile._headers});
         timeout += singleFile._fileSize;
     }

src/libsync/configfile.cpp

@@ -102,6 +102,7 @@ static constexpr char useNewBigFolderSizeLimitC[] = "useNewBigFolderSizeLimit";
 static constexpr char notifyExistingFoldersOverLimitC[] = "notifyExistingFoldersOverLimit";
 static constexpr char stopSyncingExistingFoldersOverLimitC[] = "stopSyncingExistingFoldersOverLimit";
 static constexpr char confirmExternalStorageC[] = "confirmExternalStorage";
+static constexpr char synchronizeSymlinksC[] = "synchronizeSymlinks";
 static constexpr char moveToTrashC[] = "moveToTrash";
 
 static constexpr char certPath[] = "http_certificatePath";
@@ -947,6 +948,12 @@ bool ConfigFile::confirmExternalStorage() const
     return getPolicySetting(QLatin1String(confirmExternalStorageC), fallback).toBool();
 }
 
+bool ConfigFile::synchronizeSymlinks() const
+{
+    const auto fallback = getValue(synchronizeSymlinksC, QString(), false);
+    return getPolicySetting(QLatin1String(synchronizeSymlinksC), fallback).toBool();
+}
+
 bool ConfigFile::useNewBigFolderSizeLimit() const
 {
     const auto fallback = getValue(useNewBigFolderSizeLimitC, QString(), true);
@@ -981,6 +988,11 @@ void ConfigFile::setConfirmExternalStorage(bool isChecked)
     setValue(confirmExternalStorageC, isChecked);
 }
 
+void ConfigFile::setSynchronizeSymlinks(bool isChecked)
+{
+    setValue(synchronizeSymlinksC, isChecked);
+}
+
 bool ConfigFile::moveToTrash() const
 {
     return getValue(moveToTrashC, QString(), false).toBool();

src/libsync/configfile.h

@@ -149,6 +149,9 @@ class OWNCLOUDSYNC_EXPORT ConfigFile
     [[nodiscard]] bool confirmExternalStorage() const;
     void setConfirmExternalStorage(bool);
 
+    [[nodiscard]] bool synchronizeSymlinks() const;
+    void setSynchronizeSymlinks(bool);
+
     /** If we should move the files deleted on the server in the trash  */
     [[nodiscard]] bool moveToTrash() const;
     void setMoveToTrash(bool);