QT/Serial-Studio
1
0
Fork 0
mirror of https://github.com/Serial-Studio/Serial-Studio.git synced 2025-01-15 05:22:53 +08:00
Code Issues Projects Releases Wiki Activity

Rewrite CSV module

Browse Source
This commit is contained in:
Alex Spataru 2021-12-06 19:30:00 -06:00
parent 9da34ac622
commit a78b52134f
9 changed files with 144 additions and 354 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
src/CSV/Export.cpp
View File

@ -24,18 +24,14 @@
#include <AppInfo.h>
#include <IO/Manager.h>
#include <JSON/Generator.h>
#include <JSON/FrameInfo.h>
#include <UI/Dashboard.h>
#include <Misc/Utilities.h>
#include <Misc/TimerEvents.h>
#include <QDir>
#include <QUrl>
#include <QProcess>
#include <QFileInfo>
#include <QMessageBox>
#include <QApplication>
#include <QJsonDocument>
#include <QDesktopServices>
namespace CSV
@ -50,11 +46,10 @@ Export::Export()
: m_exportEnabled(true)
{
const auto io = IO::Manager::getInstance();
const auto ge = JSON::Generator::getInstance();
const auto te = Misc::TimerEvents::getInstance();
connect(io, &IO::Manager::connectedChanged, this, &Export::closeFile);
connect(io, &IO::Manager::frameReceived, this, &Export::registerFrame);
connect(te, &Misc::TimerEvents::lowFreqTimeout, this, &Export::writeValues);
connect(ge, &JSON::Generator::jsonChanged, this, &Export::registerFrame);
}
/**
@ -114,7 +109,7 @@ void Export::setExportEnabled(const bool enabled)
if (!exportEnabled() && isOpen())
{
m_jsonList.clear();
m_frames.clear();
closeFile();
}
}
@ -126,7 +121,7 @@ void Export::closeFile()
{
if (isOpen())
{
while (m_jsonList.count())
while (m_frames.count())
writeValues();
m_csvFile.close();
@ -142,86 +137,55 @@ void Export::closeFile()
*/
void Export::writeValues()
{
// Sort JSON frames so that they are ordered from least-recent to most-recent
JFI_SortList(&m_jsonList);
// Get separator sequence
const auto sep = IO::Manager::getInstance()->separatorSequence();
// Export JSON frames
for (int k = 0; k < m_jsonList.count(); ++k)
// Write each frame
for (int i = 0; i < m_frames.count(); ++i)
{
// k is unused, since we are removing each JSON structure
// as we are exporting the file
(void)k;
// Get project title & cell values
auto dateTime = m_jsonList.first().rxDateTime;
auto json = m_jsonList.first().jsonDocument.object();
auto projectTitle = JFI_Value(json, "title", "t").toString();
// Validate JSON & title
if (json.isEmpty() || projectTitle.isEmpty())
{
m_jsonList.removeFirst();
break;
}
// Get cell titles & values
StringList titles;
StringList values;
auto groups = JFI_Value(json, "groups", "g").toArray();
for (int i = 0; i < groups.count(); ++i)
{
// Get group & dataset array
auto group = groups.at(i).toObject();
auto datasets = JFI_Value(group, "datasets", "d").toArray();
if (group.isEmpty() || datasets.isEmpty())
continue;
// Get group title
auto groupTitle = JFI_Value(group, "title", "t").toVariant().toString();
// Get dataset titles & values
for (int j = 0; j < datasets.count(); ++j)
{
// Get dataset object & fields
auto dataset = datasets.at(j).toObject();
auto datasetTitle = JFI_Value(dataset, "title", "t").toString();
auto datasetUnits = JFI_Value(dataset, "units", "u").toString();
auto datasetValue = JFI_Value(dataset, "value", "v").toString();
// Construct dataset title from group, dataset title & units
QString title;
if (datasetUnits.isEmpty())
title = QString("(%1) %2").arg(groupTitle, datasetTitle);
else
title = QString("(%1) %2 [%3]")
.arg(groupTitle, datasetTitle, datasetUnits);
// Add dataset title & value to lists
titles.append(title);
values.append(datasetValue);
}
}
// Abort if cell titles are empty
if (titles.isEmpty())
{
m_jsonList.removeFirst();
break;
}
// Prepend current time
titles.prepend("RX Date/Time");
values.prepend(dateTime.toString("yyyy/MM/dd/ HH:mm:ss::zzz"));
auto frame = m_frames.at(i);
auto fields = QString::fromUtf8(frame.data).split(sep);
// File not open, create it & add cell titles
if (!isOpen() && exportEnabled())
createCsvFile(frame);
// Write RX date/time
m_textStream << frame.rxDateTime.toString("yyyy/MM/dd/ HH:mm:ss::zzz") << ",";
// Write frame data
for (int j = 0; j < fields.count(); ++j)
{
// Get file name and path
const QString format = dateTime.toString("yyyy/MMM/dd/");
const QString fileName = dateTime.toString("HH-mm-ss") + ".csv";
const QString path = QString("%1/Documents/%2/CSV/%3/%4")
.arg(QDir::homePath(), qApp->applicationName(),
m_textStream << fields.at(j);
if (j < fields.count() - 1)
m_textStream << ",";
else
m_textStream << "\n";
}
}
// Clear frames
m_frames.clear();
}
/**
* Creates a new CSV file corresponding to the current project title & field count
*/
void Export::createCsvFile(const RawFrame &frame)
{
// Get project title
const auto projectTitle = UI::Dashboard::getInstance()->title();
// Get file name
const QString fileName = frame.rxDateTime.toString("HH-mm-ss") + ".csv";
// Get path
// clang-format off
const QString format = frame.rxDateTime.toString("yyyy/MMM/dd/");
const QString path = QString("%1/Documents/%2/CSV/%3/%4").arg(QDir::homePath(),
qApp->applicationName(),
projectTitle, format);
// clang-format on
// Generate file path if required
QDir dir(path);
@ -232,25 +196,32 @@ void Export::writeValues()
m_csvFile.setFileName(dir.filePath(fileName));
if (!m_csvFile.open(QIODevice::WriteOnly | QIODevice::Text))
{
QMessageBox::critical(Q_NULLPTR, tr("CSV File Error"),
tr("Cannot open CSV file for writing!"),
QMessageBox::Ok);
Misc::Utilities::showMessageBox(tr("CSV File Error"),
tr("Cannot open CSV file for writing!"));
closeFile();
return;
}
// Add cell titles & force UTF-8 codec
m_textStream.setDevice(&m_csvFile);
m_textStream.setGenerateByteOrderMark(true);
#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
m_textStream.setCodec("UTF-8");
#else
m_textStream.setEncoding(QStringConverter::Utf8);
#endif
m_textStream.setGenerateByteOrderMark(true);
for (int i = 0; i < titles.count(); ++i)
// Get number of fields
const auto sep = IO::Manager::getInstance()->separatorSequence();
const auto fields = QString::fromUtf8(frame.data).split(sep);
// Add table titles
m_textStream << "RX Date/Time,";
for (int j = 0; j < fields.count(); ++j)
{
m_textStream << titles.at(i).toUtf8();
if (i < titles.count() - 1)
m_textStream << "Field " << j + 1;
if (j < fields.count() - 1)
m_textStream << ",";
else
m_textStream << "\n";
@ -260,37 +231,24 @@ void Export::writeValues()
emit openChanged();
}
// Write cell values
for (int i = 0; i < values.count(); ++i)
{
m_textStream << values.at(i).toUtf8();
if (i < values.count() - 1)
m_textStream << ",";
else
m_textStream << "\n";
}
// Remove JSON from list
m_jsonList.removeFirst();
}
}
/**
* Obtains the latest JSON dataframe & appends it to the JSON list, which is later read,
* sorted and written to the CSV file by the @c writeValues() function.
* Appends the latest data from the device to the output buffer
*/
void Export::registerFrame(const JFI_Object &info)
void Export::registerFrame(const QByteArray &data)
{
// Ignore if device is not connected (we don't want to generate a CSV file when we
// are reading another CSV file don't we?)
if (!IO::Manager::getInstance()->connected())
return;
// Ignore is CSV export is disabled
// Ignore if CSV export is disabled
if (!exportEnabled())
return;
// Update JSON list
m_jsonList.append(info);
// Register raw frame to list
RawFrame frame;
frame.data = data;
frame.rxDateTime = QDateTime::currentDateTime();
m_frames.append(frame);
}
}

14
src/CSV/Export.h
View File

@ -27,14 +27,13 @@
#include <QVariant>
#include <QTextStream>
#include <QJsonObject>
#include <JSON/FrameInfo.h>
namespace CSV
{
/**
* @brief The Export class
*
* The CSV export class receives data from the @c JSON::Generator class and
* The CSV export class receives data from the @c IO::Manager class and
* exports the received frames into a CSV file selected by the user.
*
* CSV-data is generated periodically each time the @c Misc::TimerEvents
@ -42,6 +41,12 @@ namespace CSV
* is to allow exporting data, but avoid freezing the application when serial
* data is received continuously.
*/
typedef struct
{
QByteArray data;
QDateTime rxDateTime;
} RawFrame;
class Export : public QObject
{
// clang-format off
@ -76,12 +81,13 @@ public slots:
private slots:
void writeValues();
void registerFrame(const JFI_Object &info);
void createCsvFile(const RawFrame &frame);
void registerFrame(const QByteArray &data);
private:
QFile m_csvFile;
bool m_exportEnabled;
QTextStream m_textStream;
QVector<JFI_Object> m_jsonList;
QVector<RawFrame> m_frames;
};
}

206
src/CSV/Player.cpp
View File

@ -29,14 +29,8 @@
#include <qtcsv/stringdata.h>
#include <qtcsv/reader.h>
#include <QJsonValue>
#include <QJsonArray>
#include <QJsonObject>
#include <IO/Manager.h>
#include <Misc/Utilities.h>
#include <JSON/Generator.h>
#include <JSON/FrameInfo.h>
namespace CSV
{
@ -136,8 +130,10 @@ QString Player::timestamp() const
QString Player::csvFilesPath() const
{
// Get file name and path
QString path
= QString("%1/Documents/%2/CSV/").arg(QDir::homePath(), qApp->applicationName());
// clang-format off
QString path = QString("%1/Documents/%2/CSV/").arg(QDir::homePath(),
qApp->applicationName());
// clang-format on
// Generate file path if required
QDir dir(path);
@ -204,11 +200,9 @@ void Player::openFile()
void Player::closeFile()
{
m_framePos = 0;
m_model.clear();
m_csvFile.close();
m_csvData.clear();
m_playing = false;
m_datasetIndexes.clear();
m_timestamp = "--.--";
emit openChanged();
@ -247,18 +241,6 @@ void Player::previousFrame()
*/
void Player::openFile(const QString &filePath)
{
// Check that manual JSON mode is activaded
const auto opMode = JSON::Generator::getInstance()->operationMode();
const auto jsonOpen = !JSON::Generator::getInstance()->jsonMapData().isEmpty();
if (opMode != JSON::Generator::kManual || !jsonOpen)
{
Misc::Utilities::showMessageBox(
tr("Invalid configuration for CSV player"),
tr("You need to select a JSON map file in order to use "
"this feature"));
return;
}
// File name empty, abort
if (filePath.isEmpty())
return;
@ -392,11 +374,8 @@ void Player::updateData()
emit timestampChanged();
}
// Construct JSON from CSV & instruct the parser to use this document as
// input source for the QML bridge
auto json = getJsonFrame(framePosition() + 1);
if (!json.isEmpty())
JSON::Generator::getInstance()->loadJSON(json);
// Construct frame from CSV and send it to the IO manager
IO::Manager::getInstance()->processPayload(getFrame(framePosition() + 1));
// If the user wants to 'play' the CSV, get time difference between this
// frame and the next frame & schedule an automated update
@ -416,8 +395,13 @@ void Player::updateData()
const auto currDateTime = QDateTime::fromString(currTime, format);
const auto nextDateTime = QDateTime::fromString(nextTime, format);
const auto msecsToNextF = currDateTime.msecsTo(nextDateTime);
QTimer::singleShot(msecsToNextF, Qt::PreciseTimer, this,
// clang-format off
QTimer::singleShot(msecsToNextF,
Qt::PreciseTimer,
this,
SLOT(nextFrame()));
// clang-format on
}
// Error - pause playback
@ -473,149 +457,29 @@ bool Player::validateRow(const int position)
}
/**
* Generates a JSON data frame by combining the values of the current CSV
* row & the structure of the JSON map file loaded in the @c JsonParser class.
*
* The details of how this is done are a bit fuzzy, and the methods used here
* are pretty ugly & unorthodox, but they work. Brutality works.
* Generates a frame from the data at the given @a row. The first item of each row is
* ignored because it contains the RX date/time, which is used to regulate the interval
* at which the frames are parsed.
*/
QJsonDocument Player::getJsonFrame(const int row)
QByteArray Player::getFrame(const int row)
{
// Create the group/dataset model only one time
if (m_model.isEmpty())
QByteArray frame;
const auto sep = IO::Manager::getInstance()->separatorSequence();
if (m_csvData.count() > row)
{
auto titles = m_csvData.at(0);
for (int i = 1; i < titles.count(); ++i)
auto list = m_csvData.at(row);
for (int i = 1; i < list.count(); ++i)
{
// Construct group string
QString group;
const auto title = titles.at(i);
const auto glist = title.split(")");
for (int j = 0; j < glist.count() - 1; ++j)
group.append(glist.at(j));
// Remove the '(' from group name
if (!group.isEmpty())
group.remove(0, 1);
// Get dataset name & remove units
QString dataset = glist.last();
if (dataset.endsWith("]"))
{
while (!dataset.endsWith("["))
dataset.chop(1);
}
// Remove extra spaces from dataset
while (dataset.startsWith(" "))
dataset.remove(0, 1);
while (dataset.endsWith(" ") || dataset.endsWith("["))
dataset.chop(1);
// Register group with dataset map
if (!m_model.contains(group))
{
QSet<QString> set;
set.insert(dataset);
m_model.insert(group, set);
}
// Update existing group/dataset model
else if (!m_model.value(group).contains(dataset))
{
auto set = m_model.value(group);
if (!set.contains(dataset))
set.insert(dataset);
m_model.remove(group);
m_model.insert(group, set);
}
// Register dataset index with group key
if (!m_datasetIndexes.contains(group))
{
QMap<QString, int> map;
map.insert(dataset, i);
m_datasetIndexes.insert(group, map);
}
// Register dataset index with existing group key
frame.append(list.at(i).toUtf8());
if (i < list.count() - 1)
frame.append(sep.toUtf8());
else
{
auto map = m_datasetIndexes.value(group);
map.insert(dataset, i);
m_datasetIndexes.remove(group);
m_datasetIndexes.insert(group, map);
}
}
}
// Check that row is valid
if (m_csvData.count() <= row)
return QJsonDocument();
// Read CSV row & JSON template from JSON parser
const auto values = m_csvData.at(row);
const auto mapData = JSON::Generator::getInstance()->jsonMapData();
const QJsonDocument jsonTemplate = QJsonDocument::fromJson(mapData.toUtf8());
// Replace JSON title
auto json = jsonTemplate.object();
if (json.contains("t"))
json["t"] = tr("Replay of %1").arg(filename());
else
json["title"] = tr("Replay of %1").arg(filename());
// Replace values in JSON with values in row using the model.
// This is very ugly code, somebody please fix it :(
auto groups = JFI_Value(json, "groups", "g").toArray();
foreach (auto groupKey, m_model.keys())
{
for (int i = 0; i < groups.count(); ++i)
{
auto group = groups.at(i).toObject();
if (JFI_Value(group, "title", "t") == groupKey)
{
const auto datasetKeys = m_model.value(groupKey);
auto datasets = JFI_Value(group, "datasets", "d").toArray();
foreach (auto datasetKey, datasetKeys)
{
for (int j = 0; j < datasets.count(); ++j)
{
auto dataset = datasets.at(j).toObject();
if (JFI_Value(dataset, "title", "t") == datasetKey)
{
auto index = getDatasetIndex(groupKey, datasetKey);
if (values.count() > index)
{
const auto value = values.at(index);
dataset.remove("v");
dataset.remove("value");
dataset.insert("value", value);
frame.append('\n');
}
}
datasets.replace(j, dataset);
}
}
group.remove("d");
group.remove("datasets");
group.insert("datasets", datasets);
}
groups.replace(i, group);
}
}
// Update groups from JSON
json.remove("g");
json.remove("groups");
json.insert("groups", groups);
// Return new JSON document
return QJsonDocument(json);
return frame;
}
/**
@ -638,20 +502,4 @@ QString Player::getCellValue(const int row, const int column, bool &error)
error = true;
return "";
}
/**
* Returns the column/index for the dataset key that belongs to the given
* group key.
*/
int Player::getDatasetIndex(const QString &groupKey, const QString &datasetKey)
{
if (m_datasetIndexes.contains(groupKey))
{
auto map = m_datasetIndexes.value(groupKey);
if (map.contains(datasetKey))
return map.value(datasetKey);
}
return 0;
}
}

13
src/CSV/Player.h
View File

@ -22,12 +22,10 @@
#pragma once
#include <QMap>
#include <QSet>
#include <QFile>
#include <QTimer>
#include <QObject>
#include <QJsonDocument>
#include <QVector>
namespace CSV
{
@ -35,9 +33,7 @@ namespace CSV
* @brief The Player class
*
* The CSV player class allows users to select a CSV file and "re-play" it
* with Serial Studio. To do this, the user must specify an appropiate JSON
* project file to generate the equivalent frames that where received when
* the CSV file was generated.
* with Serial Studio.
*/
class Player : public QObject
{
@ -93,9 +89,8 @@ private slots:
private:
bool validateRow(const int row);
QJsonDocument getJsonFrame(const int row);
QByteArray getFrame(const int row);
QString getCellValue(const int row, const int column, bool &error);
int getDatasetIndex(const QString &groupKey, const QString &datasetKey);
private:
int m_framePos;
@ -104,7 +99,5 @@ private:
QTimer m_frameTimer;
QString m_timestamp;
QVector<QVector<QString>> m_csvData;
QMap<QString, QSet<QString>> m_model;
QMap<QString, QMap<QString, int>> m_datasetIndexes;
};
}

9
src/JSON/Editor.cpp
View File

@ -1194,8 +1194,7 @@ void Editor::setDatasetLED(const int group, const int dataset, const bool genera
* @param group index of the group in which the dataset belongs
* @param dataset index of the dataset
*/
void Editor::setDatasetGraph(const int group, const int dataset,
const bool generateGraph)
void Editor::setDatasetGraph(const int group, const int dataset, const bool generateGraph)
{
auto set = getDataset(group, dataset);
if (set)
@ -1211,8 +1210,7 @@ void Editor::setDatasetGraph(const int group, const int dataset,
* @param group index of the group in which the dataset belongs
* @param dataset index of the dataset
*/
void Editor::setDatasetFftPlot(const int group, const int dataset,
const bool generateFft)
void Editor::setDatasetFftPlot(const int group, const int dataset, const bool generateFft)
{
auto set = getDataset(group, dataset);
if (set)
@ -1228,8 +1226,7 @@ void Editor::setDatasetFftPlot(const int group, const int dataset,
* @param group index of the group in which the dataset belongs
* @param dataset index of the dataset
*/
void Editor::setDatasetLogPlot(const int group, const int dataset,
const bool generateLog)
void Editor::setDatasetLogPlot(const int group, const int dataset, const bool generateLog)
{
auto set = getDataset(group, dataset);
if (set)

15
src/JSON/Editor.h
View File

@ -156,16 +156,11 @@ public slots:
void setDatasetGraph(const int group, const int dataset, const bool generateGraph);
void setDatasetFftPlot(const int group, const int dataset, const bool generateFft);
void setDatasetLogPlot(const int group, const int dataset, const bool generateLog);
void setDatasetWidgetMin(const int group, const int dataset,
const QString &minimum);
void setDatasetWidgetMax(const int group, const int dataset,
const QString &maximum);
void setDatasetWidgetData(const int group, const int dataset,
const QString &widget);
void setDatasetWidgetAlarm(const int group, const int dataset,
const QString &alarm);
void setDatasetFFTSamples(const int group, const int dataset,
const QString &samples);
void setDatasetWidgetMin(const int group, const int dataset, const QString &minimum);
void setDatasetWidgetMax(const int group, const int dataset, const QString &maximum);
void setDatasetWidgetData(const int group, const int dataset, const QString &widget);
void setDatasetWidgetAlarm(const int group, const int dataset, const QString &alarm);
void setDatasetFFTSamples(const int group, const int dataset, const QString &samples);
private slots:
void onJsonLoaded();

7
src/JSON/Generator.cpp
View File

@ -289,10 +289,6 @@ void Generator::reset()
*/
void Generator::readData(const QByteArray &data)
{
// CSV-replay active, abort
if (CSV::Player::getInstance()->isOpen())
return;
// Data empty, abort
if (data.isEmpty())
return;
@ -402,8 +398,7 @@ void Generator::processFrame(const QByteArray &data, const quint64 frame,
* Constructor function, stores received frame data & the date/time that the frame data
* was received.
*/
JSONWorker::JSONWorker(const QByteArray &data, const quint64 frame,
const QDateTime &time)
JSONWorker::JSONWorker(const QByteArray &data, const quint64 frame, const QDateTime &time)
: m_time(time)
, m_data(data)
, m_frame(frame)

3
src/JSON/Generator.h
View File

@ -146,8 +146,7 @@ public slots:
private slots:
void reset();
void readData(const QByteArray &data);
void processFrame(const QByteArray &data, const quint64 frame,
const QDateTime &time);
void processFrame(const QByteArray &data, const quint64 frame, const QDateTime &time);
private:
QFile m_jsonMap;

3
src/UI/Dashboard.cpp
View File

@ -1035,8 +1035,7 @@ bool Dashboard::getVisibility(const QVector<bool> &vector, const int index) cons
* vector. Calling this function with @a visible set to @c false will hide the widget in
* the QML user interface.
*/
void Dashboard::setVisibility(QVector<bool> &vector, const int index,
const bool visible)
void Dashboard::setVisibility(QVector<bool> &vector, const int index, const bool visible)
{
if (index < vector.count())
{