From a78b52134ff618775868b679b91b1aa5f7150930 Mon Sep 17 00:00:00 2001 From: Alex Spataru Date: Mon, 6 Dec 2021 19:30:00 -0600 Subject: [PATCH] Rewrite CSV module --- src/CSV/Export.cpp | 228 +++++++++++++++++------------------------ src/CSV/Export.h | 14 ++- src/CSV/Player.cpp | 206 +++++-------------------------------- src/CSV/Player.h | 13 +-- src/JSON/Editor.cpp | 9 +- src/JSON/Editor.h | 15 +-- src/JSON/Generator.cpp | 7 +- src/JSON/Generator.h | 3 +- src/UI/Dashboard.cpp | 3 +- 9 files changed, 144 insertions(+), 354 deletions(-) diff --git a/src/CSV/Export.cpp b/src/CSV/Export.cpp index 0dfc3c3b..a8211f19 100644 --- a/src/CSV/Export.cpp +++ b/src/CSV/Export.cpp @@ -24,18 +24,14 @@ #include #include -#include -#include +#include #include #include #include #include -#include #include -#include #include -#include #include namespace CSV @@ -50,11 +46,10 @@ Export::Export() : m_exportEnabled(true) { const auto io = IO::Manager::getInstance(); - const auto ge = JSON::Generator::getInstance(); const auto te = Misc::TimerEvents::getInstance(); connect(io, &IO::Manager::connectedChanged, this, &Export::closeFile); + connect(io, &IO::Manager::frameReceived, this, &Export::registerFrame); connect(te, &Misc::TimerEvents::lowFreqTimeout, this, &Export::writeValues); - connect(ge, &JSON::Generator::jsonChanged, this, &Export::registerFrame); } /** @@ -114,7 +109,7 @@ void Export::setExportEnabled(const bool enabled) if (!exportEnabled() && isOpen()) { - m_jsonList.clear(); + m_frames.clear(); closeFile(); } } @@ -126,7 +121,7 @@ void Export::closeFile() { if (isOpen()) { - while (m_jsonList.count()) + while (m_frames.count()) writeValues(); m_csvFile.close(); @@ -142,155 +137,118 @@ void Export::closeFile() */ void Export::writeValues() { - // Sort JSON frames so that they are ordered from least-recent to most-recent - JFI_SortList(&m_jsonList); + // Get separator sequence + const auto sep = IO::Manager::getInstance()->separatorSequence(); - // Export JSON frames - for (int k = 0; k < m_jsonList.count(); ++k) + // Write each frame + for (int i = 0; i < m_frames.count(); ++i) { - // k is unused, since we are removing each JSON structure - // as we are exporting the file - (void)k; - - // Get project title & cell values - auto dateTime = m_jsonList.first().rxDateTime; - auto json = m_jsonList.first().jsonDocument.object(); - auto projectTitle = JFI_Value(json, "title", "t").toString(); - - // Validate JSON & title - if (json.isEmpty() || projectTitle.isEmpty()) - { - m_jsonList.removeFirst(); - break; - } - - // Get cell titles & values - StringList titles; - StringList values; - auto groups = JFI_Value(json, "groups", "g").toArray(); - for (int i = 0; i < groups.count(); ++i) - { - // Get group & dataset array - auto group = groups.at(i).toObject(); - auto datasets = JFI_Value(group, "datasets", "d").toArray(); - if (group.isEmpty() || datasets.isEmpty()) - continue; - - // Get group title - auto groupTitle = JFI_Value(group, "title", "t").toVariant().toString(); - - // Get dataset titles & values - for (int j = 0; j < datasets.count(); ++j) - { - // Get dataset object & fields - auto dataset = datasets.at(j).toObject(); - auto datasetTitle = JFI_Value(dataset, "title", "t").toString(); - auto datasetUnits = JFI_Value(dataset, "units", "u").toString(); - auto datasetValue = JFI_Value(dataset, "value", "v").toString(); - - // Construct dataset title from group, dataset title & units - QString title; - if (datasetUnits.isEmpty()) - title = QString("(%1) %2").arg(groupTitle, datasetTitle); - else - title = QString("(%1) %2 [%3]") - .arg(groupTitle, datasetTitle, datasetUnits); - - // Add dataset title & value to lists - titles.append(title); - values.append(datasetValue); - } - } - - // Abort if cell titles are empty - if (titles.isEmpty()) - { - m_jsonList.removeFirst(); - break; - } - - // Prepend current time - titles.prepend("RX Date/Time"); - values.prepend(dateTime.toString("yyyy/MM/dd/ HH:mm:ss::zzz")); + auto frame = m_frames.at(i); + auto fields = QString::fromUtf8(frame.data).split(sep); // File not open, create it & add cell titles if (!isOpen() && exportEnabled()) + createCsvFile(frame); + + // Write RX date/time + m_textStream << frame.rxDateTime.toString("yyyy/MM/dd/ HH:mm:ss::zzz") << ","; + + // Write frame data + for (int j = 0; j < fields.count(); ++j) { - // Get file name and path - const QString format = dateTime.toString("yyyy/MMM/dd/"); - const QString fileName = dateTime.toString("HH-mm-ss") + ".csv"; - const QString path = QString("%1/Documents/%2/CSV/%3/%4") - .arg(QDir::homePath(), qApp->applicationName(), - projectTitle, format); - - // Generate file path if required - QDir dir(path); - if (!dir.exists()) - dir.mkpath("."); - - // Open file - m_csvFile.setFileName(dir.filePath(fileName)); - if (!m_csvFile.open(QIODevice::WriteOnly | QIODevice::Text)) - { - QMessageBox::critical(Q_NULLPTR, tr("CSV File Error"), - tr("Cannot open CSV file for writing!"), - QMessageBox::Ok); - closeFile(); - return; - } - - // Add cell titles & force UTF-8 codec - m_textStream.setDevice(&m_csvFile); -#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) - m_textStream.setCodec("UTF-8"); -#else - m_textStream.setEncoding(QStringConverter::Utf8); -#endif - m_textStream.setGenerateByteOrderMark(true); - for (int i = 0; i < titles.count(); ++i) - { - m_textStream << titles.at(i).toUtf8(); - if (i < titles.count() - 1) - m_textStream << ","; - else - m_textStream << "\n"; - } - - // Update UI - emit openChanged(); - } - - // Write cell values - for (int i = 0; i < values.count(); ++i) - { - m_textStream << values.at(i).toUtf8(); - if (i < values.count() - 1) + m_textStream << fields.at(j); + if (j < fields.count() - 1) m_textStream << ","; else m_textStream << "\n"; } - - // Remove JSON from list - m_jsonList.removeFirst(); } + + // Clear frames + m_frames.clear(); } /** - * Obtains the latest JSON dataframe & appends it to the JSON list, which is later read, - * sorted and written to the CSV file by the @c writeValues() function. + * Creates a new CSV file corresponding to the current project title & field count */ -void Export::registerFrame(const JFI_Object &info) +void Export::createCsvFile(const RawFrame &frame) +{ + // Get project title + const auto projectTitle = UI::Dashboard::getInstance()->title(); + + // Get file name + const QString fileName = frame.rxDateTime.toString("HH-mm-ss") + ".csv"; + + // Get path + // clang-format off + const QString format = frame.rxDateTime.toString("yyyy/MMM/dd/"); + const QString path = QString("%1/Documents/%2/CSV/%3/%4").arg(QDir::homePath(), + qApp->applicationName(), + projectTitle, format); + // clang-format on + + // Generate file path if required + QDir dir(path); + if (!dir.exists()) + dir.mkpath("."); + + // Open file + m_csvFile.setFileName(dir.filePath(fileName)); + if (!m_csvFile.open(QIODevice::WriteOnly | QIODevice::Text)) + { + Misc::Utilities::showMessageBox(tr("CSV File Error"), + tr("Cannot open CSV file for writing!")); + closeFile(); + return; + } + + // Add cell titles & force UTF-8 codec + m_textStream.setDevice(&m_csvFile); + m_textStream.setGenerateByteOrderMark(true); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) + m_textStream.setCodec("UTF-8"); +#else + m_textStream.setEncoding(QStringConverter::Utf8); +#endif + + // Get number of fields + const auto sep = IO::Manager::getInstance()->separatorSequence(); + const auto fields = QString::fromUtf8(frame.data).split(sep); + + // Add table titles + m_textStream << "RX Date/Time,"; + for (int j = 0; j < fields.count(); ++j) + { + m_textStream << "Field " << j + 1; + + if (j < fields.count() - 1) + m_textStream << ","; + else + m_textStream << "\n"; + } + + // Update UI + emit openChanged(); +} + +/** + * Appends the latest data from the device to the output buffer + */ +void Export::registerFrame(const QByteArray &data) { // Ignore if device is not connected (we don't want to generate a CSV file when we // are reading another CSV file don't we?) if (!IO::Manager::getInstance()->connected()) return; - // Ignore is CSV export is disabled + // Ignore if CSV export is disabled if (!exportEnabled()) return; - // Update JSON list - m_jsonList.append(info); + // Register raw frame to list + RawFrame frame; + frame.data = data; + frame.rxDateTime = QDateTime::currentDateTime(); + m_frames.append(frame); } } diff --git a/src/CSV/Export.h b/src/CSV/Export.h index 7cd1af39..0b9cdf22 100644 --- a/src/CSV/Export.h +++ b/src/CSV/Export.h @@ -27,14 +27,13 @@ #include #include #include -#include namespace CSV { /** * @brief The Export class * - * The CSV export class receives data from the @c JSON::Generator class and + * The CSV export class receives data from the @c IO::Manager class and * exports the received frames into a CSV file selected by the user. * * CSV-data is generated periodically each time the @c Misc::TimerEvents @@ -42,6 +41,12 @@ namespace CSV * is to allow exporting data, but avoid freezing the application when serial * data is received continuously. */ +typedef struct +{ + QByteArray data; + QDateTime rxDateTime; +} RawFrame; + class Export : public QObject { // clang-format off @@ -76,12 +81,13 @@ public slots: private slots: void writeValues(); - void registerFrame(const JFI_Object &info); + void createCsvFile(const RawFrame &frame); + void registerFrame(const QByteArray &data); private: QFile m_csvFile; bool m_exportEnabled; QTextStream m_textStream; - QVector m_jsonList; + QVector m_frames; }; } diff --git a/src/CSV/Player.cpp b/src/CSV/Player.cpp index 877e5116..c9d57f37 100644 --- a/src/CSV/Player.cpp +++ b/src/CSV/Player.cpp @@ -29,14 +29,8 @@ #include #include -#include -#include -#include - #include #include -#include -#include namespace CSV { @@ -136,8 +130,10 @@ QString Player::timestamp() const QString Player::csvFilesPath() const { // Get file name and path - QString path - = QString("%1/Documents/%2/CSV/").arg(QDir::homePath(), qApp->applicationName()); + // clang-format off + QString path = QString("%1/Documents/%2/CSV/").arg(QDir::homePath(), + qApp->applicationName()); + // clang-format on // Generate file path if required QDir dir(path); @@ -204,11 +200,9 @@ void Player::openFile() void Player::closeFile() { m_framePos = 0; - m_model.clear(); m_csvFile.close(); m_csvData.clear(); m_playing = false; - m_datasetIndexes.clear(); m_timestamp = "--.--"; emit openChanged(); @@ -247,18 +241,6 @@ void Player::previousFrame() */ void Player::openFile(const QString &filePath) { - // Check that manual JSON mode is activaded - const auto opMode = JSON::Generator::getInstance()->operationMode(); - const auto jsonOpen = !JSON::Generator::getInstance()->jsonMapData().isEmpty(); - if (opMode != JSON::Generator::kManual || !jsonOpen) - { - Misc::Utilities::showMessageBox( - tr("Invalid configuration for CSV player"), - tr("You need to select a JSON map file in order to use " - "this feature")); - return; - } - // File name empty, abort if (filePath.isEmpty()) return; @@ -392,11 +374,8 @@ void Player::updateData() emit timestampChanged(); } - // Construct JSON from CSV & instruct the parser to use this document as - // input source for the QML bridge - auto json = getJsonFrame(framePosition() + 1); - if (!json.isEmpty()) - JSON::Generator::getInstance()->loadJSON(json); + // Construct frame from CSV and send it to the IO manager + IO::Manager::getInstance()->processPayload(getFrame(framePosition() + 1)); // If the user wants to 'play' the CSV, get time difference between this // frame and the next frame & schedule an automated update @@ -416,8 +395,13 @@ void Player::updateData() const auto currDateTime = QDateTime::fromString(currTime, format); const auto nextDateTime = QDateTime::fromString(nextTime, format); const auto msecsToNextF = currDateTime.msecsTo(nextDateTime); - QTimer::singleShot(msecsToNextF, Qt::PreciseTimer, this, + + // clang-format off + QTimer::singleShot(msecsToNextF, + Qt::PreciseTimer, + this, SLOT(nextFrame())); + // clang-format on } // Error - pause playback @@ -473,149 +457,29 @@ bool Player::validateRow(const int position) } /** - * Generates a JSON data frame by combining the values of the current CSV - * row & the structure of the JSON map file loaded in the @c JsonParser class. - * - * The details of how this is done are a bit fuzzy, and the methods used here - * are pretty ugly & unorthodox, but they work. Brutality works. + * Generates a frame from the data at the given @a row. The first item of each row is + * ignored because it contains the RX date/time, which is used to regulate the interval + * at which the frames are parsed. */ -QJsonDocument Player::getJsonFrame(const int row) +QByteArray Player::getFrame(const int row) { - // Create the group/dataset model only one time - if (m_model.isEmpty()) + QByteArray frame; + const auto sep = IO::Manager::getInstance()->separatorSequence(); + + if (m_csvData.count() > row) { - auto titles = m_csvData.at(0); - for (int i = 1; i < titles.count(); ++i) + auto list = m_csvData.at(row); + for (int i = 1; i < list.count(); ++i) { - // Construct group string - QString group; - const auto title = titles.at(i); - const auto glist = title.split(")"); - for (int j = 0; j < glist.count() - 1; ++j) - group.append(glist.at(j)); - - // Remove the '(' from group name - if (!group.isEmpty()) - group.remove(0, 1); - - // Get dataset name & remove units - QString dataset = glist.last(); - if (dataset.endsWith("]")) - { - while (!dataset.endsWith("[")) - dataset.chop(1); - } - - // Remove extra spaces from dataset - while (dataset.startsWith(" ")) - dataset.remove(0, 1); - while (dataset.endsWith(" ") || dataset.endsWith("[")) - dataset.chop(1); - - // Register group with dataset map - if (!m_model.contains(group)) - { - QSet set; - set.insert(dataset); - m_model.insert(group, set); - } - - // Update existing group/dataset model - else if (!m_model.value(group).contains(dataset)) - { - auto set = m_model.value(group); - if (!set.contains(dataset)) - set.insert(dataset); - - m_model.remove(group); - m_model.insert(group, set); - } - - // Register dataset index with group key - if (!m_datasetIndexes.contains(group)) - { - QMap map; - map.insert(dataset, i); - m_datasetIndexes.insert(group, map); - } - - // Register dataset index with existing group key + frame.append(list.at(i).toUtf8()); + if (i < list.count() - 1) + frame.append(sep.toUtf8()); else - { - auto map = m_datasetIndexes.value(group); - map.insert(dataset, i); - m_datasetIndexes.remove(group); - m_datasetIndexes.insert(group, map); - } + frame.append('\n'); } } - // Check that row is valid - if (m_csvData.count() <= row) - return QJsonDocument(); - - // Read CSV row & JSON template from JSON parser - const auto values = m_csvData.at(row); - const auto mapData = JSON::Generator::getInstance()->jsonMapData(); - const QJsonDocument jsonTemplate = QJsonDocument::fromJson(mapData.toUtf8()); - - // Replace JSON title - auto json = jsonTemplate.object(); - if (json.contains("t")) - json["t"] = tr("Replay of %1").arg(filename()); - else - json["title"] = tr("Replay of %1").arg(filename()); - - // Replace values in JSON with values in row using the model. - // This is very ugly code, somebody please fix it :( - auto groups = JFI_Value(json, "groups", "g").toArray(); - foreach (auto groupKey, m_model.keys()) - { - for (int i = 0; i < groups.count(); ++i) - { - auto group = groups.at(i).toObject(); - - if (JFI_Value(group, "title", "t") == groupKey) - { - const auto datasetKeys = m_model.value(groupKey); - auto datasets = JFI_Value(group, "datasets", "d").toArray(); - foreach (auto datasetKey, datasetKeys) - { - for (int j = 0; j < datasets.count(); ++j) - { - auto dataset = datasets.at(j).toObject(); - if (JFI_Value(dataset, "title", "t") == datasetKey) - { - auto index = getDatasetIndex(groupKey, datasetKey); - if (values.count() > index) - { - const auto value = values.at(index); - dataset.remove("v"); - dataset.remove("value"); - dataset.insert("value", value); - } - } - - datasets.replace(j, dataset); - } - } - - group.remove("d"); - group.remove("datasets"); - group.insert("datasets", datasets); - } - - groups.replace(i, group); - } - } - - // Update groups from JSON - json.remove("g"); - json.remove("groups"); - json.insert("groups", groups); - - // Return new JSON document - return QJsonDocument(json); + return frame; } /** @@ -638,20 +502,4 @@ QString Player::getCellValue(const int row, const int column, bool &error) error = true; return ""; } - -/** - * Returns the column/index for the dataset key that belongs to the given - * group key. - */ -int Player::getDatasetIndex(const QString &groupKey, const QString &datasetKey) -{ - if (m_datasetIndexes.contains(groupKey)) - { - auto map = m_datasetIndexes.value(groupKey); - if (map.contains(datasetKey)) - return map.value(datasetKey); - } - - return 0; -} } diff --git a/src/CSV/Player.h b/src/CSV/Player.h index db451f40..2fcd3c97 100644 --- a/src/CSV/Player.h +++ b/src/CSV/Player.h @@ -22,12 +22,10 @@ #pragma once -#include -#include #include #include #include -#include +#include namespace CSV { @@ -35,9 +33,7 @@ namespace CSV * @brief The Player class * * The CSV player class allows users to select a CSV file and "re-play" it - * with Serial Studio. To do this, the user must specify an appropiate JSON - * project file to generate the equivalent frames that where received when - * the CSV file was generated. + * with Serial Studio. */ class Player : public QObject { @@ -93,9 +89,8 @@ private slots: private: bool validateRow(const int row); - QJsonDocument getJsonFrame(const int row); + QByteArray getFrame(const int row); QString getCellValue(const int row, const int column, bool &error); - int getDatasetIndex(const QString &groupKey, const QString &datasetKey); private: int m_framePos; @@ -104,7 +99,5 @@ private: QTimer m_frameTimer; QString m_timestamp; QVector> m_csvData; - QMap> m_model; - QMap> m_datasetIndexes; }; } diff --git a/src/JSON/Editor.cpp b/src/JSON/Editor.cpp index 27ce7f45..4bc05e12 100644 --- a/src/JSON/Editor.cpp +++ b/src/JSON/Editor.cpp @@ -1194,8 +1194,7 @@ void Editor::setDatasetLED(const int group, const int dataset, const bool genera * @param group index of the group in which the dataset belongs * @param dataset index of the dataset */ -void Editor::setDatasetGraph(const int group, const int dataset, - const bool generateGraph) +void Editor::setDatasetGraph(const int group, const int dataset, const bool generateGraph) { auto set = getDataset(group, dataset); if (set) @@ -1211,8 +1210,7 @@ void Editor::setDatasetGraph(const int group, const int dataset, * @param group index of the group in which the dataset belongs * @param dataset index of the dataset */ -void Editor::setDatasetFftPlot(const int group, const int dataset, - const bool generateFft) +void Editor::setDatasetFftPlot(const int group, const int dataset, const bool generateFft) { auto set = getDataset(group, dataset); if (set) @@ -1228,8 +1226,7 @@ void Editor::setDatasetFftPlot(const int group, const int dataset, * @param group index of the group in which the dataset belongs * @param dataset index of the dataset */ -void Editor::setDatasetLogPlot(const int group, const int dataset, - const bool generateLog) +void Editor::setDatasetLogPlot(const int group, const int dataset, const bool generateLog) { auto set = getDataset(group, dataset); if (set) diff --git a/src/JSON/Editor.h b/src/JSON/Editor.h index 791cb6a2..b744bd9e 100644 --- a/src/JSON/Editor.h +++ b/src/JSON/Editor.h @@ -156,16 +156,11 @@ public slots: void setDatasetGraph(const int group, const int dataset, const bool generateGraph); void setDatasetFftPlot(const int group, const int dataset, const bool generateFft); void setDatasetLogPlot(const int group, const int dataset, const bool generateLog); - void setDatasetWidgetMin(const int group, const int dataset, - const QString &minimum); - void setDatasetWidgetMax(const int group, const int dataset, - const QString &maximum); - void setDatasetWidgetData(const int group, const int dataset, - const QString &widget); - void setDatasetWidgetAlarm(const int group, const int dataset, - const QString &alarm); - void setDatasetFFTSamples(const int group, const int dataset, - const QString &samples); + void setDatasetWidgetMin(const int group, const int dataset, const QString &minimum); + void setDatasetWidgetMax(const int group, const int dataset, const QString &maximum); + void setDatasetWidgetData(const int group, const int dataset, const QString &widget); + void setDatasetWidgetAlarm(const int group, const int dataset, const QString &alarm); + void setDatasetFFTSamples(const int group, const int dataset, const QString &samples); private slots: void onJsonLoaded(); diff --git a/src/JSON/Generator.cpp b/src/JSON/Generator.cpp index 61386c89..ab45c9e3 100644 --- a/src/JSON/Generator.cpp +++ b/src/JSON/Generator.cpp @@ -289,10 +289,6 @@ void Generator::reset() */ void Generator::readData(const QByteArray &data) { - // CSV-replay active, abort - if (CSV::Player::getInstance()->isOpen()) - return; - // Data empty, abort if (data.isEmpty()) return; @@ -402,8 +398,7 @@ void Generator::processFrame(const QByteArray &data, const quint64 frame, * Constructor function, stores received frame data & the date/time that the frame data * was received. */ -JSONWorker::JSONWorker(const QByteArray &data, const quint64 frame, - const QDateTime &time) +JSONWorker::JSONWorker(const QByteArray &data, const quint64 frame, const QDateTime &time) : m_time(time) , m_data(data) , m_frame(frame) diff --git a/src/JSON/Generator.h b/src/JSON/Generator.h index a6c39e99..804ca905 100644 --- a/src/JSON/Generator.h +++ b/src/JSON/Generator.h @@ -146,8 +146,7 @@ public slots: private slots: void reset(); void readData(const QByteArray &data); - void processFrame(const QByteArray &data, const quint64 frame, - const QDateTime &time); + void processFrame(const QByteArray &data, const quint64 frame, const QDateTime &time); private: QFile m_jsonMap; diff --git a/src/UI/Dashboard.cpp b/src/UI/Dashboard.cpp index ed225232..24cc2ca1 100644 --- a/src/UI/Dashboard.cpp +++ b/src/UI/Dashboard.cpp @@ -1035,8 +1035,7 @@ bool Dashboard::getVisibility(const QVector &vector, const int index) cons * vector. Calling this function with @a visible set to @c false will hide the widget in * the QML user interface. */ -void Dashboard::setVisibility(QVector &vector, const int index, - const bool visible) +void Dashboard::setVisibility(QVector &vector, const int index, const bool visible) { if (index < vector.count()) {