chore: add .exe installer on Windows

.github/workflows/ubuntu.yml

@@ -42,6 +42,9 @@ jobs:
             sudo apt install libmpfr-dev #required by rmpfr packages
             sudo apt install libglpk-dev #required by igraph packages
             sudo apt install jags
+            sudo apt install libminizip-dev # required by freexl
+            git clone https://github.com/jasp-stats/freexl.git
+            cd freexl && ./configure && make && sudo make install
 
       - name: Install boost
         uses: MarkusJx/[email protected]

.gitmodules

@@ -150,3 +150,11 @@
 	path = Common/jaspColumnEncoder
 	url = https://github.com/jasp-stats/jaspColumnEncoder
 	branch = master
+[submodule "Engine/jaspModuleInstaller"]
+	path = Engine/jaspModuleInstaller
+	url = https://github.com/jasp-stats/jaspModuleInstaller.git
+	branch = master
+[submodule "Modules/jaspBFF"]
+	path = Modules/jaspBFF
+	url = https://github.com/jasp-stats/jaspBFF
+	branch = master

Common/utilenums.h

@@ -2,7 +2,7 @@
 #define UTILENUMS_H
 #include "enumutilities.h"
 
-DECLARE_ENUM(FileTypeBase,	jasp = 0, html, csv, txt, tsv, sav, zsav, ods, pdf, sas7bdat, sas7bcat, por, xpt, dta, database, empty, unknown );
+DECLARE_ENUM(FileTypeBase,	jasp = 0, html, csv, txt, tsv, sav, zsav, ods, xls, xlsx, pdf, sas7bdat, sas7bcat, por, xpt, dta, database, empty, unknown );
 
 //const QStringList Database::dbTypes() const should be updated if DbType is changed.
 DECLARE_ENUM(DbType,		NOTCHOSEN, QDB2, /*QIBASE,*/ QMYSQL, QOCI, QODBC, QPSQL, QSQLITE /*, QSQLITE2, QTDS*/ );

CommonData/column.cpp

@@ -24,7 +24,10 @@ Column::Column(DataSet * data, int id)
 	_emptyValues(		new EmptyValues(data->emptyValues())),
 	_doubleDummy(		new Label(this)),
 	_autoSortByValue(	_autoSortByValuesByDefault)
-{}
+{
+	if(_id != -1)
+		db().columnSetAutoSort(_id, _autoSortByValue); //Store autosort in db
+}
 
 Column::~Column()
 {
@@ -82,7 +85,7 @@ void Column::dbDelete(bool cleanUpRest)
 {
 	assert(_id != -1);
 
-	labelsClear();
+	labelsClear(false);
 	db().columnDelete(_id, cleanUpRest);
 
 	_id = -1;
@@ -649,13 +652,14 @@ void Column::_sortLabelsByOrder()
 	std::sort(_labels.begin(), _labels.end(), [](const Label * l, const Label * r) { return l->order() < r->order(); });
 }
 
-void Column::labelsClear()
+void Column::labelsClear(bool doIncRevision)
 {
 	db().labelsClear(_id);
 	_labels.clear();
 	_labelByIntsIdMap.clear();
 
-	incRevision(false);
+	if(doIncRevision)
+		incRevision(false);
 }
 
 void Column::beginBatchedLabelsDB()

CommonData/column.h

@@ -109,7 +109,7 @@ class Column : public DataSetBaseNode
 			void					upgradeSetDoubleLabelsInInts();			///< Used by upgrade 0.18.* -> 0.19
 			void					upgradeExtractDoublesIntsFromLabels();	///< Used by upgrade 0.18.* -> 0.19
 
-			void					labelsClear();
+			void					labelsClear(bool doIncRevision=true);
 			int						labelsAdd(			int display);
 			int						labelsAdd(			const std::string & display);
 			int						labelsAdd(			const std::string & display, const std::string & description, const Json::Value & originalValue);

CommonData/databaseinterface.cpp

@@ -75,7 +75,7 @@ int DatabaseInterface::dataSetInsert(const std::string & dataFilePath, long data
 
 	transactionWriteBegin();
 	int id = runStatementsId("INSERT INTO DataSets (dataFilePath, dataFileTimestamp, description, databaseJson, emptyValuesJson, dataFileSynch) VALUES (?, ?, ?, ?, ?, ?) RETURNING id;", prepare);
-	runStatements("CREATE TABLE " + dataSetName(id) + " (rowNumber INTEGER PRIMARY KEY);");
+	runStatements("CREATE TABLE " + dataSetName(id) + " (rowNumber INTEGER PRIMARY KEY);"); // Can be overwritten through dataSetCreateTable
 	transactionWriteEnd();
 
 	return id;
@@ -183,6 +183,7 @@ void DatabaseInterface::filterDelete(int filterIndex)
 
 	if(dataSetId != -1)
 		runStatements("ALTER TABLE " + dataSetName(dataSetId) + " DROP COLUMN " + filterName(filterIndex) + ";");
+
 	runStatements("DELETE FROM Filters WHERE id = " + std::to_string(filterIndex) + ";");
 
 	transactionWriteEnd();
@@ -391,7 +392,7 @@ void DatabaseInterface::filterWrite(int filterIndex, const std::vector<bool> & v
 	transactionWriteEnd();
 }
 
-int DatabaseInterface::columnInsert(int dataSetId, int index, const std::string & name, columnType colType)
+int DatabaseInterface::columnInsert(int dataSetId, int index, const std::string & name, columnType colType, bool alterTable)
 {
 	JASPTIMER_SCOPE(DatabaseInterface::columnInsert);
 	transactionWriteBegin();
@@ -419,19 +420,38 @@ int DatabaseInterface::columnInsert(int dataSetId, int index, const std::string
 		Log::log() << "Inserting column failed!" << std::endl;
 #endif
 
-	//Add a scalar and ordinal/nominal column to DataSet_# for the column
-	const std::string alterDatasetPrefix = "ALTER TABLE " + dataSetName(dataSetId);
-	const std::string addColumnFragment  = " ADD  " + columnBaseName(columnId);
-
-	runStatements(alterDatasetPrefix + addColumnFragment + "_DBL REAL NULL;");
-	runStatements(alterDatasetPrefix + addColumnFragment + "_INT INT  NULL;");
-
+
+	if(alterTable) //If not then via dataSetCreateTable
+	{
+		//Add a scalar and ordinal/nominal column to DataSet_# for the column
+		const std::string alterDatasetPrefix = "ALTER TABLE " + dataSetName(dataSetId);
+		const std::string addColumnFragment  = " ADD  " + columnBaseName(columnId);
+
+		runStatements(alterDatasetPrefix + addColumnFragment + "_DBL REAL NULL;");
+		runStatements(alterDatasetPrefix + addColumnFragment + "_INT INT  NULL;");
+	}
+
 	//The labels will be added separately later
 
 	transactionWriteEnd();
 	return columnId;
 }
 
+void DatabaseInterface::dataSetCreateTable(DataSet * dataSet)
+{
+	runStatements("DROP TABLE " + dataSetName(dataSet->id()) + ";");
+
+	std::stringstream statements;
+	statements <<  "CREATE TABLE " + dataSetName(dataSet->id()) + " (rowNumber INTEGER PRIMARY KEY, "+ filterName(dataSet->filter()->id()) + " INT NOT NULL DEFAULT 1";
+
+	for(Column * column : dataSet->columns())
+		statements << ", " << columnBaseName(column->id()) << "_DBL REAL NULL, " << columnBaseName(column->id()) << "_INT INT NULL";
+
+	statements << ");";
+
+	runStatements(statements.str());
+}
+
 int DatabaseInterface::columnGetDataSetId(int columnId)
 {
 	JASPTIMER_SCOPE(DatabaseInterface::columnGetDataSetId);
@@ -1438,19 +1458,19 @@ void DatabaseInterface::_runStatements(const std::string & statements, bindParam
 	}
 	while(remain > 1 && (ret == SQLITE_OK && ret != SQLITE_DONE));
 
+	const int maxLenStatementError = 200;
+	std::string shortStatements = statements.size() <= maxLenStatementError ? statements : statements.substr(0, maxLenStatementError);
+
 	if(ret == SQLITE_ERROR)
 	{
-		std::string errorMsg = "Running ```\n"+statements+"\n``` failed because of: `" + sqlite3_errmsg(_db);
-		Log::log() << errorMsg << std::endl;
-
-		throw std::runtime_error(errorMsg);
+		Log::log() <<				"Running ```\n"+statements		+"\n``` failed because of: `" + sqlite3_errmsg(_db) << std::endl;
+		throw std::runtime_error(	"Running ```\n"+shortStatements	+"\n``` failed because of: `" + sqlite3_errmsg(_db));
 	}
 
 	if(ret == SQLITE_READONLY)
 	{
-		std::string errorMsg = "Running ```\n"+statements+"\n``` failed because the database is readonly...";
-		Log::log() << errorMsg << std::endl;
-		throw std::runtime_error(errorMsg);
+		Log::log() <<				"Running ```\n"+statements		+"\n``` failed because the database is readonly..." << std::endl;
+		throw std::runtime_error(	"Running ```\n"+shortStatements	+"\n``` failed because the database is readonly...");
 	}
 }
 
@@ -1569,8 +1589,7 @@ void DatabaseInterface::create()
 	}
 	else
 		Log::log() << "Opened internal sqlite database for creation at '" << dbFile() << "'." << std::endl;
-
-
+
 	transactionWriteBegin();
 	runStatements(_dbConstructionSql);
 	transactionWriteEnd();
@@ -1593,6 +1612,7 @@ void DatabaseInterface::load()
 	}
 	else
 		Log::log() << "Opened internal sqlite database for loading at '" << dbFile() << "'." << std::endl;
+
 }
 
 void DatabaseInterface::close()
@@ -1662,3 +1682,4 @@ void DatabaseInterface::transactionReadEnd()
 		runStatements("COMMIT");
 }
 
+

CommonData/databaseinterface.h

@@ -87,6 +87,7 @@ class DatabaseInterface
 	int			dataSetGetRevision(		int dataSetId);
 	int			dataSetGetFilter(		int dataSetId);
 	void		dataSetInsertEmptyRow(	int dataSetId, size_t row);
+	void		dataSetCreateTable(		DataSet * dataSet); ///< Assumes you are importing fresh data and havent created any DataSet_? table yet
 
 	void		dataSetBatchedValuesUpdate(DataSet * data, std::vector<Column*> columns, std::function<void(float)> progressCallback = [](float){});
 	void		dataSetBatchedValuesUpdate(DataSet * data, std::function<void(float)> progressCallback = [](float){});
@@ -109,7 +110,7 @@ class DatabaseInterface
 
 	//Columns & Data/Values
 	//Index stuff:
-	int			columnInsert(			int dataSetId, int index = -1, const std::string & name = "", columnType colType = columnType::unknown);	///< Insert a row into Columns and create the corresponding columns in DataSet_? Also makes sure the indices are correct
+	int			columnInsert(			int dataSetId, int index = -1, const std::string & name = "", columnType colType = columnType::unknown, bool alterTable=true);	///< Insert a row into Columns and create the corresponding columns in DataSet_? Also makes sure the indices are correct
 	int			columnLastFreeIndex(	int dataSetId);
 	void		columnIndexIncrements(	int dataSetId, int index);																			///< If index already is in use that column and all after are incremented by 1
 	void		columnIndexDecrements(	int dataSetId, int index);																			///< Indices bigger than index are decremented, assumption is that the previous one using it has been removed already
@@ -157,7 +158,7 @@ class DatabaseInterface
 	void		transactionWriteEnd(bool rollback = false);		///< runs COMMIT or ROLLBACK based on rollback and ends the transaction.  Tracks whether nested and only does BEGIN+COMMIT at lowest depth
 	void		transactionReadBegin();							///< runs BEGIN DEFERRED and waits for sqlite to not be busy anymore if some other process is writing  Tracks whether nested and only does BEGIN+COMMIT at lowest depth
 	void		transactionReadEnd();							///< runs COMMIT and ends the transaction. Tracks whether nested and only does BEGIN+COMMIT at lowest depth
-
+		
 private:
 	void		_doubleTroubleBinder(sqlite3_stmt *stmt, int param, double dbl);	///< Needed to work around the lack of support for NAN, INF and NEG_INF in sqlite, converts those to string to make use of sqlite flexibility
 	double		_doubleTroubleReader(sqlite3_stmt *stmt, int colI);					///< The reading counterpart to _doubleTroubleBinder to convert string representations of NAN, INF and NEG_INF back to double

CommonData/dataset.cpp

@@ -59,6 +59,7 @@ void DataSet::dbDelete()
 
 	_dataSetID = -1;
 
+
 	db().transactionWriteEnd();
 }
 
@@ -171,12 +172,12 @@ void DataSet::removeColumn(const std::string & name)
 		}
 }
 
-void DataSet::insertColumn(size_t index)
+void DataSet::insertColumn(size_t index,	bool alterDataSetTable)
 {
 
 	assert(_dataSetID > 0);
 
-	Column * newColumn = new Column(this, db().columnInsert(_dataSetID, index));
+	Column * newColumn = new Column(this, db().columnInsert(_dataSetID, index, "", columnType::unknown, alterDataSetTable));
 
 	_columns.insert(_columns.begin()+index, newColumn);
 
@@ -382,18 +383,24 @@ void DataSet::setColumnCount(size_t colCount)
 	db().transactionWriteBegin();
 
 	int curCount = columns().size();
+
+	bool alterTableAfterwards = curCount == 0 && colCount > 0;
 
 	if(colCount > curCount)
 		for(size_t i=curCount; i<colCount; i++)
-			insertColumn(i);
+			insertColumn(i, !alterTableAfterwards);
 
 	else if(colCount < curCount)
 		for(size_t i=curCount-1; i>=colCount; i--)
 			removeColumn(i);
 
+
 	incRevision();
 
 	db().transactionWriteEnd();
+
+	if(alterTableAfterwards)
+		db().dataSetCreateTable(this);
 }
 
 void DataSet::setRowCount(size_t rowCount)

CommonData/dataset.h

@@ -40,11 +40,11 @@ class DataSet : public DataSetBaseNode
 			void			beginBatchedToDB();
 			void			endBatchedToDB(std::function<void(float)> progressCallback = [](float){}, Columns columns={});
 			void			endBatchedToDB(Columns columns) { endBatchedToDB([](float){}, columns); }
-
+			
 			void			removeColumn(	const	std::string &	name	);
 			void			removeColumn(			size_t			index	);
 			void			removeColumnById(		size_t			id		);
-			void			insertColumn(			size_t			index	);
+			void			insertColumn(			size_t			index,	bool alterDataSetTable = true);
 			Column		*	newColumn(		const	std::string &	name);
 			int				getColumnIndex(	const	std::string &	name	) const;
 			int				columnIndex(	const	Column		*	col		) const;

Desktop/CMakeLists.txt

@@ -42,7 +42,7 @@ qt_add_executable(
   ${BUNDLE_RESOURCES}
   $<$<PLATFORM_ID:Windows>:${CMAKE_CURRENT_LIST_DIR}/icon.rc>
   $<$<PLATFORM_ID:Darwin>:${_R_Framework}>
-  $<$<PLATFORM_ID:Windows>:${CMAKE_SOURCE_DIR}/Desktop/JASP.exe.manifest>  
+  $<$<PLATFORM_ID:Windows>:${CMAKE_SOURCE_DIR}/Desktop/JASP.exe.manifest>
 )
 
 set(
@@ -105,6 +105,7 @@ target_include_directories(
     $<$<BOOL:${FLATPAK_USED}>:/app/include/QtCore5Compat>
     $<$<BOOL:${FLATPAK_USED}>:/app/include/QtWebEngineQuick>
     $<$<BOOL:${FLATPAK_USED}>:/app/include/QtWebEngineCore>
+    ${LIBFREEXL_INCLUDE_DIRS}
 )
 
 target_link_libraries(
@@ -145,6 +146,9 @@ target_link_libraries(
     Iconv::Iconv
     OpenSSL::SSL
     OpenSSL::Crypto
+    # FreeXL
+    ${LIBFREEXL_LIBRARIES}
+    $<$<NOT:$<BOOL:${LINUX_LOCAL_BUILD}>>:freexl::freexl>
     # ReadStat -----------------------------------
     ${LIBREADSTAT_LIBRARIES}
     # MinGW's ReadStat

Desktop/components/JASP/Widgets/ComputeColumnWindow.qml

@@ -2,6 +2,7 @@ import QtQuick			2.7
 import QtQuick.Controls 2.12
 import QtQuick.Layouts	1.3
 import JASP.Controls	1.0 as JaspControls
+import JASP
 
 import "FilterConstructor"
 
@@ -103,6 +104,12 @@ FocusScope
 				TextArea
 				{
 					id: computeColumnEdit
+
+					RSyntaxHighlighterQuick
+					{
+						textDocument:		computeColumnEdit.textDocument
+					}
+
 
 					anchors.top:			parent.top
 					anchors.left:			parent.left

Desktop/components/JASP/Widgets/DataPanel.qml

@@ -22,7 +22,7 @@ Rectangle
 			id:							filterWindow
 			objectName:					"filterWindow"
 			SplitView.minimumHeight:	desiredMinimumHeight
-			SplitView.preferredHeight:	rootDataset.height * 0.25
+			SplitView.preferredHeight:	desiredHeight
 			SplitView.maximumHeight:	rootDataset.height * 0.8
 
 		}