JDierkse/DataStorageInterface
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Makefile.conf and fix Line Endings

Browse Source
This commit is contained in:
JDierkse
2020-05-04 21:45:25 +02:00
parent 9b66a7dd1f
commit 94739511dd
15 changed files with 952 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Application/DataImporter.cc
+89 -89
View File
@@ -1,89 +1,89 @@
#include "DataInterface/DataImporter.h" #include "DataInterface/DataImporter.h"
#include <clipp.h> #include <clipp.h>
#include <INIReader.h> #include <INIReader.h>
#include <Logging.h> #include <Logging.h>
#include <MySQLClient.h> #include <MySQLClient.h>
#include <SQLiteClient.h> #include <SQLiteClient.h>
#include <fstream> #include <fstream>
#include <iostream> #include <iostream>
#include <sstream> #include <sstream>
int main(int argc, char** argv) int main(int argc, char** argv)
{ {
try try
{ {
Logging::OpenLog(); Logging::OpenLog();
Logging::SetLogMask(Logging::Severity::Debug); Logging::SetLogMask(Logging::Severity::Debug);
std::string databaseFile = ""; std::string databaseFile = "";
clipp::group cli { clipp::group cli {
clipp::value("SQLite Database file", databaseFile) clipp::value("SQLite Database file", databaseFile)
}; };
if (!clipp::parse(argc, argv, cli)) if (!clipp::parse(argc, argv, cli))
{ {
std::stringstream ss; std::stringstream ss;
ss << clipp::make_man_page(cli, argv[0]); ss << clipp::make_man_page(cli, argv[0]);
Logging::Log(Logging::Severity::Info, ss.str()); Logging::Log(Logging::Severity::Info, ss.str());
Logging::CloseLog(); Logging::CloseLog();
return 1; return 1;
} }
{ {
std::ifstream f(databaseFile); std::ifstream f(databaseFile);
if (!f.good()) if (!f.good())
throw std::runtime_error("Can't read SQLite Database file."); throw std::runtime_error("Can't read SQLite Database file.");
} }
INIReader iniReader("DataImporter.ini"); INIReader iniReader("DataImporter.ini");
if (iniReader.ParseError() != 0) if (iniReader.ParseError() != 0)
throw std::runtime_error("Can't read DataImporter.ini."); throw std::runtime_error("Can't read DataImporter.ini.");
MySQL::MySQLClient mySQLClient; MySQL::MySQLClient mySQLClient;
{ {
std::string hostname = iniReader.Get("MySQL", "Hostname", ""); std::string hostname = iniReader.Get("MySQL", "Hostname", "");
if (hostname.empty()) if (hostname.empty())
throw std::runtime_error("MySQL Hostname directive missing in ini file."); throw std::runtime_error("MySQL Hostname directive missing in ini file.");
std::string username = iniReader.Get("MySQL", "Username", ""); std::string username = iniReader.Get("MySQL", "Username", "");
if (username.empty()) if (username.empty())
throw std::runtime_error("MySQL Username directive missing in ini file."); throw std::runtime_error("MySQL Username directive missing in ini file.");
std::string password = iniReader.Get("MySQL", "Password", ""); std::string password = iniReader.Get("MySQL", "Password", "");
if (password.empty()) if (password.empty())
throw std::runtime_error("MySQL Password directive missing in ini file."); throw std::runtime_error("MySQL Password directive missing in ini file.");
std::string database = iniReader.Get("MySQL", "Database", ""); std::string database = iniReader.Get("MySQL", "Database", "");
if (database.empty()) if (database.empty())
throw std::runtime_error("MySQL Database directive missing in ini file."); throw std::runtime_error("MySQL Database directive missing in ini file.");
mySQLClient.Connect(hostname, username, password, database); mySQLClient.Connect(hostname, username, password, database);
} }
SQLite::SQLiteClient sqliteClient("./datalog.db"); SQLite::SQLiteClient sqliteClient("./datalog.db");
DataStorageInterface::DataInterface::DataImporter importer(&mySQLClient, &sqliteClient); DataStorageInterface::DataInterface::DataImporter importer(&mySQLClient, &sqliteClient);
importer.ImportData(); importer.ImportData();
Logging::CloseLog(); Logging::CloseLog();
} }
catch (const std::exception& e) catch (const std::exception& e)
{ {
std::cerr << "Exception caught" << std::endl; std::cerr << "Exception caught" << std::endl;
std::stringstream ss; std::stringstream ss;
ss << "Type : " << typeid(e).name() << std::endl; ss << "Type : " << typeid(e).name() << std::endl;
ss << "ERROR: " << e.what() << std::endl; ss << "ERROR: " << e.what() << std::endl;
Logging::Log(Logging::Severity::Error, ss.str()); Logging::Log(Logging::Severity::Error, ss.str());
Logging::CloseLog(); Logging::CloseLog();
return -1; return -1;
} }
return 0; return 0;
} }
Application/DataStorageAPI.cc
+77 -77
View File
@@ -1,77 +1,77 @@
#include "API/WebAPI.h" #include "API/WebAPI.h"
#include "DataInterface/GraphClient.h" #include "DataInterface/GraphClient.h"
#include <DataStorageClient.h> #include <DataStorageClient.h>
#include <HttpServer.h> #include <HttpServer.h>
#include <INIReader.h> #include <INIReader.h>
#include <Logging.h> #include <Logging.h>
#include <MySQLClient.h> #include <MySQLClient.h>
#include <sstream> #include <sstream>
int main(int argc, char** argv) int main(int argc, char** argv)
{ {
try try
{ {
Logging::OpenLog(); Logging::OpenLog();
Logging::SetLogMask(Logging::Severity::Info); Logging::SetLogMask(Logging::Severity::Info);
INIReader iniReader("DataStorageAPI.ini"); INIReader iniReader("DataStorageAPI.ini");
if (iniReader.ParseError() != 0) if (iniReader.ParseError() != 0)
throw std::runtime_error("Can't read DataLogger.ini."); throw std::runtime_error("Can't read DataLogger.ini.");
Logging::Log(Logging::Severity::Info, "Starting DataStorageAPI"); Logging::Log(Logging::Severity::Info, "Starting DataStorageAPI");
int port = iniReader.GetInteger("DataStorageAPI", "Port", 0); int port = iniReader.GetInteger("DataStorageAPI", "Port", 0);
if (port == 0) if (port == 0)
throw std::runtime_error("Port directive missing in ini file."); throw std::runtime_error("Port directive missing in ini file.");
std::string table = iniReader.Get("DataStorageAPI", "Table", ""); std::string table = iniReader.Get("DataStorageAPI", "Table", "");
if (table.empty()) if (table.empty())
throw std::runtime_error("Table directive missing in ini file."); throw std::runtime_error("Table directive missing in ini file.");
std::shared_ptr<MySQL::MySQLClient> pMySQLClient = std::make_shared<MySQL::MySQLClient>(); std::shared_ptr<MySQL::MySQLClient> pMySQLClient = std::make_shared<MySQL::MySQLClient>();
{ {
std::string hostname = iniReader.Get("MySQL", "Hostname", ""); std::string hostname = iniReader.Get("MySQL", "Hostname", "");
if (hostname.empty()) if (hostname.empty())
throw std::runtime_error("MySQL Hostname directive missing in ini file."); throw std::runtime_error("MySQL Hostname directive missing in ini file.");
std::string username = iniReader.Get("MySQL", "Username", ""); std::string username = iniReader.Get("MySQL", "Username", "");
if (username.empty()) if (username.empty())
throw std::runtime_error("MySQL Username directive missing in ini file."); throw std::runtime_error("MySQL Username directive missing in ini file.");
std::string password = iniReader.Get("MySQL", "Password", ""); std::string password = iniReader.Get("MySQL", "Password", "");
if (password.empty()) if (password.empty())
throw std::runtime_error("MySQL Password directive missing in ini file."); throw std::runtime_error("MySQL Password directive missing in ini file.");
std::string database = iniReader.Get("MySQL", "Database", ""); std::string database = iniReader.Get("MySQL", "Database", "");
if (database.empty()) if (database.empty())
throw std::runtime_error("MySQL Database directive missing in ini file."); throw std::runtime_error("MySQL Database directive missing in ini file.");
pMySQLClient->Connect(hostname, username, password, database); pMySQLClient->Connect(hostname, username, password, database);
} }
DataStorage::DataStorageClient dataStorageClient(pMySQLClient, table); DataStorage::DataStorageClient dataStorageClient(pMySQLClient, table);
DataStorageInterface::DataInterface::GraphClient graphClient(pMySQLClient.get()); DataStorageInterface::DataInterface::GraphClient graphClient(pMySQLClient.get());
auto callback = std::bind(&DataStorageInterface::API::WebAPI::ProcessQuery, &dataStorageClient, &graphClient, std::placeholders::_1, std::placeholders::_2); auto callback = std::bind(&DataStorageInterface::API::WebAPI::ProcessQuery, &dataStorageClient, &graphClient, std::placeholders::_1, std::placeholders::_2);
Http::HttpServer server(port, callback); Http::HttpServer server(port, callback);
Logging::Log(Logging::Severity::Info, "Startup Complete"); Logging::Log(Logging::Severity::Info, "Startup Complete");
server.Wait(); server.Wait();
Logging::Log(Logging::Severity::Info, "Stopping DataStorageAPI..."); Logging::Log(Logging::Severity::Info, "Stopping DataStorageAPI...");
Logging::CloseLog(); Logging::CloseLog();
} }
catch (const std::exception& e) catch (const std::exception& e)
{ {
std::stringstream ss; std::stringstream ss;
ss << "ERROR: " << e.what() << std::endl; ss << "ERROR: " << e.what() << std::endl;
Logging::Log(Logging::Severity::Error, ss.str()); Logging::Log(Logging::Severity::Error, ss.str());
Logging::CloseLog(); Logging::CloseLog();
return -1; return -1;
} }
return 0; return 0;
} }
DataCondenser.ini
+7 -8
View File
@@ -1,8 +1,7 @@
[DataCondenser] [DataCondenser]
[MySQL] [MySQL]
Hostname = Hostname =
Username = Username =
Password = Password =
Database = Database =
DataImporter.ini
+7 -8
View File
@@ -1,8 +1,7 @@
[DataImporter] [DataImporter]
[MySQL] [MySQL]
Hostname = Hostname =
Username = Username =
Password = Password =
Database = Database =
DataInterface/DataCondenser.cpp
+369 -407
View File
@@ -1,407 +1,369 @@
#include "DataCondenser.h" #include "DataCondenser.h"
#include "Util/Util.h" #include <DataId.h>
#include <DataId.h> #include <DateTime.h>
#include <iostream> #include <limits>
#include <limits> #include <map>
#include <map> #include <memory>
#include <memory> #include <sstream>
#include <sstream>
namespace DataStorageInterface {
namespace DataStorageInterface { namespace DataInterface {
namespace DataInterface {
DataCondenser::DataCondenser(MySQL::MySQLClient* pMySQLClient) :
DataCondenser::DataCondenser(MySQL::MySQLClient* pMySQLClient) : m_pMySQLClient(pMySQLClient)
m_pMySQLClient(pMySQLClient) {
{ }
}
void DataCondenser::CondenseData()
void DataCondenser::CondenseData() {
{ auto deviceIDs = GetDeviceIDs();
auto deviceIDs = GetDeviceIDs();
for (int dataId = DataStorage::DataId::Min(); dataId <= DataStorage::DataId::Max(); ++dataId)
for (int dataId = DataStorage::DataId::Min(); dataId <= DataStorage::DataId::Max(); ++dataId) {
{ //for (auto& deviceId : deviceIDs)
std::cout << "DataID: " << dataId << std::endl; // WriteReducedData(deviceId, dataId, RRA::Daily);
for (auto& deviceId : deviceIDs)
//std::cout << "Daily" << std::endl; WriteReducedData(deviceId, dataId, RRA::Weekly);
//for (auto& deviceId : deviceIDs) for (auto& deviceId : deviceIDs)
// WriteReducedData(deviceId, dataId, RRA::Daily); WriteReducedData(deviceId, dataId, RRA::Monthly);
std::cout << "Weekly" << std::endl; for (auto& deviceId : deviceIDs)
for (auto& deviceId : deviceIDs) WriteReducedData(deviceId, dataId, RRA::Yearly);
WriteReducedData(deviceId, dataId, RRA::Weekly); }
std::cout << "Monthly" << std::endl;
for (auto& deviceId : deviceIDs) CleanData();
WriteReducedData(deviceId, dataId, RRA::Monthly); }
std::cout << "Yearly" << std::endl;
for (auto& deviceId : deviceIDs) int DataCondenser::GetPeriodStamp(int timestamp, const RRA::type& rra)
WriteReducedData(deviceId, dataId, RRA::Yearly); {
} int modulo = timestamp % static_cast<int>(rra);
return timestamp - modulo;
CleanData(); }
}
std::vector<int> DataCondenser::GetDeviceIDs()
int DataCondenser::GetPeriodStamp(int timestamp, const RRA::type& rra) {
{ std::stringstream query;
int modulo = timestamp % static_cast<int>(rra); query << "SELECT DISTINCT `device_id` FROM `datalog` ORDER BY `device_id`;";
return timestamp - modulo; auto resultSet = m_pMySQLClient->ExecuteQuery(query.str());
}
std::vector<int> returnValue;
std::vector<int> DataCondenser::GetDeviceIDs() while (resultSet.Next())
{ returnValue.push_back(resultSet.Int("device_id"));
std::stringstream query;
query << "SELECT DISTINCT `device_id` FROM `datalog` ORDER BY `device_id`;"; return returnValue;
auto resultSet = m_pMySQLClient->ExecuteQuery(query.str()); }
std::vector<int> returnValue; int DataCondenser::GetNewestPeriodStamp(const std::string& sourceTable, int deviceId, int dataId)
while (resultSet.Next()) {
returnValue.push_back(resultSet.Int("device_id")); std::stringstream query;
query << "SELECT MAX(UNIX_TIMESTAMP(`timestamp`)) AS timestamp FROM `" << sourceTable << "` WHERE `device_id` = '" << deviceId <<"' AND `data_id` = '" << dataId << "';";
return returnValue; auto resultSet = m_pMySQLClient->ExecuteQuery(query.str());
} if (resultSet.Next())
return resultSet.Int("timestamp");
int DataCondenser::GetNewestPeriodStamp(const std::string& sourceTable, int deviceId, int dataId)
{ return 0;
std::stringstream query; }
query << "SELECT MAX(UNIX_TIMESTAMP(`timestamp`)) AS timestamp FROM `" << sourceTable << "` WHERE `device_id` = '" << deviceId <<"' AND `data_id` = '" << dataId << "';";
auto resultSet = m_pMySQLClient->ExecuteQuery(query.str()); MySQL::MySQLResultSet DataCondenser::GetRawData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp)
if (resultSet.Next()) {
return resultSet.Int("timestamp"); std::stringstream query;
query << "SELECT UNIX_TIMESTAMP(`timestamp`) AS timestamp, `value` AS `min-value`, `value` AS `mean-value`, `value` AS `max-value` FROM `" << sourceTable << "` WHERE `device_id` = '" << deviceId <<"' AND `data_id` = '" << dataId << "' ";
return 0; if (newestPeriodStamp == -1)
} query << "LIMIT 1;";
else
MySQL::MySQLResultSet DataCondenser::GetRawData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp) query << "AND `timestamp` > FROM_UNIXTIME(" << newestPeriodStamp << ") ORDER BY `timestamp` ASC;";
{
std::stringstream query; return m_pMySQLClient->ExecuteQuery(query.str());
query << "SELECT UNIX_TIMESTAMP(`timestamp`) AS timestamp, `value` AS `min-value`, `value` AS `mean-value`, `value` AS `max-value` FROM `" << sourceTable << "` WHERE `device_id` = '" << deviceId <<"' AND `data_id` = '" << dataId << "' "; }
if (newestPeriodStamp == -1)
query << "LIMIT 1;"; MySQL::MySQLResultSet DataCondenser::GetReducedData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp)
else {
query << "AND `timestamp` > FROM_UNIXTIME(" << newestPeriodStamp << ") ORDER BY `timestamp` ASC;"; std::stringstream query;
query << "SELECT UNIX_TIMESTAMP(`timestamp`) AS timestamp, `min-value`, `mean-value`, `max-value` FROM `" << sourceTable << "` WHERE `device_id` = '" << deviceId <<"' AND `data_id` = '" << dataId << "' ";
return m_pMySQLClient->ExecuteQuery(query.str()); if (newestPeriodStamp == -1)
} query << "LIMIT 1;";
else
MySQL::MySQLResultSet DataCondenser::GetReducedData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp) query << "AND `timestamp` > FROM_UNIXTIME(" << newestPeriodStamp << ") ORDER BY `timestamp` ASC;";
{
std::stringstream query; return m_pMySQLClient->ExecuteQuery(query.str());
query << "SELECT UNIX_TIMESTAMP(`timestamp`) AS timestamp, `min-value`, `mean-value`, `max-value` FROM `" << sourceTable << "` WHERE `device_id` = '" << deviceId <<"' AND `data_id` = '" << dataId << "' "; }
if (newestPeriodStamp == -1)
query << "LIMIT 1;"; int DataCondenser::GetMaximumMySQLPacketSize()
else {
query << "AND `timestamp` > FROM_UNIXTIME(" << newestPeriodStamp << ") ORDER BY `timestamp` ASC;"; std::stringstream query;
query << "show variables like 'max_allowed_packet';";
return m_pMySQLClient->ExecuteQuery(query.str()); MySQL::MySQLResultSet resultSet = m_pMySQLClient->ExecuteQuery(query.str());
}
if (resultSet.RowsCount() != 1)
int DataCondenser::GetMaximumMySQLPacketSize() throw std::runtime_error("Unexpected reply when retrieving max_allowed_packet.");
{
std::stringstream query; resultSet.Next();
query << "show variables like 'max_allowed_packet';"; return resultSet.Int("Value");
MySQL::MySQLResultSet resultSet = m_pMySQLClient->ExecuteQuery(query.str()); }
if (resultSet.RowsCount() != 1) DataType::type DataCondenser::GetDataType(int deviceId, int dataId)
throw std::runtime_error("Unexpected reply when retrieving max_allowed_packet."); {
auto resultSet = GetRawData("datalog", deviceId, dataId, -1);
resultSet.Next(); if (resultSet.Next())
return resultSet.Int("Value"); return DataInterface::GetDataType(resultSet.String("mean-value"));
}
return DataType::None;
DataType::type DataCondenser::GetDataType(int deviceId, int dataId) }
{
auto resultSet = GetRawData("datalog", deviceId, dataId, -1); std::string DataCondenser::GetSourceTable(const RRA::type& rra)
if (resultSet.Next()) {
return DataInterface::GetDataType(resultSet.String("mean-value")); switch (rra)
{
return DataType::None; case RRA::Daily:
} return "datalog";
case RRA::Weekly:
std::string DataCondenser::GetSourceTable(const RRA::type& rra) return "datalog";
{ case RRA::Monthly:
switch (rra) return "datalog-weekly";
{ case RRA::Yearly:
case RRA::Daily: return "datalog-monthly";
return "datalog"; default:
case RRA::Weekly: case RRA::Raw:
return "datalog"; return std::string();
case RRA::Monthly: }
return "datalog-weekly"; }
case RRA::Yearly:
return "datalog-monthly"; std::string DataCondenser::GetDestinationTable(const RRA::type& rra)
default: {
case RRA::Raw: switch (rra)
return std::string(); {
} case RRA::Daily:
} return "datalog-daily";
case RRA::Weekly:
std::string DataCondenser::GetDestinationTable(const RRA::type& rra) return "datalog-weekly";
{ case RRA::Monthly:
switch (rra) return "datalog-monthly";
{ case RRA::Yearly:
case RRA::Daily: return "datalog-yearly";
return "datalog-daily"; default:
case RRA::Weekly: case RRA::Raw:
return "datalog-weekly"; return std::string();
case RRA::Monthly: }
return "datalog-monthly"; }
case RRA::Yearly:
return "datalog-yearly"; int DataCondenser::StringStreamSize(std::stringstream& ss)
default: {
case RRA::Raw: ss.seekp(0, std::ios_base::end);
return std::string(); return ss.tellp();
} }
}
void DataCondenser::InitializeInsertQuery(std::stringstream& query, const std::string& table)
int DataCondenser::StringStreamSize(std::stringstream& ss) {
{ query.str("");
ss.seekp(0, std::ios_base::end); query << "INSERT INTO `" << table << "` (`device_id`, `data_id`, `timestamp`, `min-value`, `mean-value`, `max-value`) VALUES ";
return ss.tellp(); }
}
void DataCondenser::ExecuteInsertQuery(std::stringstream& query)
void DataCondenser::InitializeInsertQuery(std::stringstream& query, const std::string& table) {
{ query.seekp(-1, std::ios_base::end);
query.str(""); query << ";";
query << "INSERT INTO `" << table << "` (`device_id`, `data_id`, `timestamp`, `min-value`, `mean-value`, `max-value`) VALUES "; m_pMySQLClient->Execute(query.str());
} }
void DataCondenser::ExecuteInsertQuery(std::stringstream& query) void DataCondenser::WriteReducedData(int deviceId, int dataId, const RRA::type& rra)
{ {
query.seekp(-1, std::ios_base::end); DataType::type dataType = GetDataType(deviceId, dataId);
query << ";";
m_pMySQLClient->Execute(query.str()); switch (dataType)
} {
case DataType::Float:
void DataCondenser::WriteReducedData(int deviceId, int dataId, const RRA::type& rra) WriteReducedFloatData(deviceId, dataId, rra);
{ break;
DataType::type dataType = GetDataType(deviceId, dataId); case DataType::String:
WriteReducedStringData(deviceId, dataId, rra);
if (dataType == DataType::None) break;
std::cout << " DataType: None" << std::endl; default:
else if (dataType == DataType::Float) case DataType::None:
std::cout << " DataType: Float" << std::endl; break;
else if (dataType == DataType::String) }
std::cout << " DataType: String" << std::endl; }
switch (dataType) void DataCondenser::WriteReducedFloatData(int deviceId, int dataId, const RRA::type& rra)
{ {
case DataType::Float: int maxMySQLPacketSize = GetMaximumMySQLPacketSize() - 2;
WriteReducedFloatData(deviceId, dataId, rra);
break; std::string sourceTable = GetSourceTable(rra);
case DataType::String: std::string destinationTable = GetDestinationTable(rra);
WriteReducedStringData(deviceId, dataId, rra);
break; int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId);
default: int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId);
case DataType::None:
break; std::stringstream query;
} MySQL::MySQLResultSet resultSet;
} if (sourceTable == "datalog")
resultSet = GetRawData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
void DataCondenser::WriteReducedFloatData(int deviceId, int dataId, const RRA::type& rra) else
{ resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
int maxMySQLPacketSize = GetMaximumMySQLPacketSize() - 2;
int samples = 0;
std::string sourceTable = GetSourceTable(rra); float min = std::numeric_limits<float>::max();
std::string destinationTable = GetDestinationTable(rra); float mean = 0;
float max = std::numeric_limits<float>::min();
int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId);
int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId); int writeCount = 0;
std::cout << " DeviceID: " << deviceId << std::endl; InitializeInsertQuery(query, destinationTable);
std::cout << " DataID: " << dataId << std::endl;
std::cout << " Newest Src : " << newestSourcePeriodStamp << std::endl; std::stringstream queryPart;
std::cout << " Newest Dest: " << newestDestinationPeriodStamp << std::endl;
int periodStamp = 0;
std::stringstream query; while (resultSet.Next())
MySQL::MySQLResultSet resultSet; {
if (sourceTable == "datalog") int currentPeriodStamp = GetPeriodStamp(resultSet.Int("timestamp"), rra);
resultSet = GetRawData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp); if (currentPeriodStamp != periodStamp)
else {
resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp); if (samples > 0)
{
std::cout << " Source Count: " << resultSet.RowsCount() << std::endl; mean = mean / samples;
int samples = 0; queryPart.str("");
float min = std::numeric_limits<float>::max(); queryPart << "('" << deviceId << "', '" << dataId << "', FROM_UNIXTIME(" << currentPeriodStamp << "), '" << min << "', '" << mean << "', '" << max << "'),";
float mean = 0; ++writeCount;
float max = std::numeric_limits<float>::min();
int packetSize = StringStreamSize(query) + StringStreamSize(queryPart);
int writeCount = 0; if (packetSize > maxMySQLPacketSize)
{
InitializeInsertQuery(query, destinationTable); ExecuteInsertQuery(query);
InitializeInsertQuery(query, destinationTable);
std::stringstream queryPart; writeCount = 0;
}
int periodStamp = 0;
while (resultSet.Next()) query << queryPart.str();
{ }
int currentPeriodStamp = GetPeriodStamp(resultSet.Int("timestamp"), rra);
if (currentPeriodStamp != periodStamp) if (currentPeriodStamp > newestSourcePeriodStamp)
{ break;
if (samples > 0)
{ samples = 0;
mean = mean / samples; min = std::numeric_limits<float>::max();
mean = 0;
queryPart.str(""); max = std::numeric_limits<float>::min();
queryPart << "('" << deviceId << "', '" << dataId << "', FROM_UNIXTIME(" << currentPeriodStamp << "), '" << min << "', '" << mean << "', '" << max << "'),"; periodStamp = currentPeriodStamp;
++writeCount; }
int packetSize = StringStreamSize(query) + StringStreamSize(queryPart); float minValue, meanValue, maxValue;
if (packetSize > maxMySQLPacketSize) minValue = static_cast<float>(resultSet.Double("min-value"));
{ meanValue = static_cast<float>(resultSet.Double("mean-value"));
ExecuteInsertQuery(query); maxValue = static_cast<float>(resultSet.Double("max-value"));
InitializeInsertQuery(query, destinationTable);
std::cout << " Write Count: " << writeCount << std::endl; if (minValue < min)
writeCount = 0; min = minValue;
} if (maxValue > max)
max = maxValue;
query << queryPart.str(); mean += meanValue;
} ++samples;
}
if (currentPeriodStamp > newestSourcePeriodStamp)
break; if (writeCount > 0)
ExecuteInsertQuery(query);
samples = 0; }
min = std::numeric_limits<float>::max();
mean = 0; void DataCondenser::WriteReducedStringData(int deviceId, int dataId, const RRA::type& rra)
max = std::numeric_limits<float>::min(); {
periodStamp = currentPeriodStamp; int maxMySQLPacketSize = GetMaximumMySQLPacketSize() - 2;
}
std::string sourceTable = GetSourceTable(rra);
float minValue, meanValue, maxValue; std::string destinationTable = GetDestinationTable(rra);
minValue = static_cast<float>(resultSet.Double("min-value"));
meanValue = static_cast<float>(resultSet.Double("mean-value")); int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId);
maxValue = static_cast<float>(resultSet.Double("max-value")); int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId);
if (minValue < min) std::stringstream query;
min = minValue; MySQL::MySQLResultSet resultSet;
if (maxValue > max) if (sourceTable == "datalog")
max = maxValue; resultSet = GetRawData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
mean += meanValue; else
++samples; resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
}
int samples = 0;
if (writeCount > 0) std::vector<std::string> values;
ExecuteInsertQuery(query);
int writeCount = 0;
std::cout << " Write Count: " << writeCount << std::endl << std::endl;
} InitializeInsertQuery(query, destinationTable);
void DataCondenser::WriteReducedStringData(int deviceId, int dataId, const RRA::type& rra) std::stringstream queryPart;
{
int maxMySQLPacketSize = GetMaximumMySQLPacketSize() - 2; int periodStamp = 0;
while (resultSet.Next())
std::string sourceTable = GetSourceTable(rra); {
std::string destinationTable = GetDestinationTable(rra); int currentPeriodStamp = GetPeriodStamp(resultSet.Int("timestamp"), rra);
if (currentPeriodStamp != periodStamp)
int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId); {
int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId); if (samples > 0)
{
std::cout << " DeviceID: " << deviceId << std::endl; std::map<std::string, int> map;
std::cout << " DataID: " << dataId << std::endl; for (auto& value : values)
std::cout << " Newest Src : " << newestSourcePeriodStamp << std::endl; {
std::cout << " Newest Dest: " << newestDestinationPeriodStamp << std::endl; auto iterator = map.find(value);
if (iterator == map.end())
std::stringstream query; map.insert(std::pair<std::string, int>(value, 1));
MySQL::MySQLResultSet resultSet; else
if (sourceTable == "datalog") map[value] += 1;
resultSet = GetRawData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp); }
else
resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp); auto iterator = map.begin();
for (std::map<std::string, int>::iterator it = map.begin(); it != map.end(); ++it)
std::cout << " Source Count: " << resultSet.RowsCount() << std::endl; {
if (it->second > iterator->second)
int samples = 0; iterator = it;
std::vector<std::string> values; }
int writeCount = 0; queryPart.str("");
queryPart << "('" << deviceId << "', '" << dataId << "', FROM_UNIXTIME(" << currentPeriodStamp << "), '', '" << iterator->first << "', ''),";
InitializeInsertQuery(query, destinationTable);
int packetSize = StringStreamSize(query) + StringStreamSize(queryPart);
std::stringstream queryPart; if (packetSize > maxMySQLPacketSize)
{
int periodStamp = 0; ExecuteInsertQuery(query);
while (resultSet.Next()) InitializeInsertQuery(query, destinationTable);
{ writeCount = 0;
int currentPeriodStamp = GetPeriodStamp(resultSet.Int("timestamp"), rra); }
if (currentPeriodStamp != periodStamp)
{ query << queryPart.str();
if (samples > 0) ++writeCount;
{ }
std::map<std::string, int> map;
for (auto& value : values) if (currentPeriodStamp > newestSourcePeriodStamp)
{ break;
auto iterator = map.find(value);
if (iterator == map.end()) samples = 0;
map.insert(std::pair<std::string, int>(value, 1)); values.clear();
else periodStamp = currentPeriodStamp;
map[value] += 1; }
}
std::string meanValue = resultSet.String("mean-value");
auto iterator = map.begin(); values.push_back(meanValue);
for (std::map<std::string, int>::iterator it = map.begin(); it != map.end(); ++it) ++samples;
{ }
if (it->second > iterator->second)
iterator = it; if (writeCount > 0)
} ExecuteInsertQuery(query);
}
queryPart.str("");
queryPart << "('" << deviceId << "', '" << dataId << "', FROM_UNIXTIME(" << currentPeriodStamp << "), '', '" << iterator->first << "', ''),"; void DataCondenser::CleanData()
{
int packetSize = StringStreamSize(query) + StringStreamSize(queryPart); CleanData(RRA::Daily);
if (packetSize > maxMySQLPacketSize) CleanData(RRA::Weekly);
{ CleanData(RRA::Monthly);
ExecuteInsertQuery(query); CleanData(RRA::Yearly);
InitializeInsertQuery(query, destinationTable); }
std::cout << " Write Count: " << writeCount << std::endl;
writeCount = 0; void DataCondenser::CleanData(const RRA::type& rra)
} {
std::string table = GetDestinationTable(rra);
query << queryPart.str(); int periodInSeconds = static_cast<int>(rra) * 400;
++writeCount; int timestamp = DateTime::GetTimestamp() + DateTime::GetUTCOffset() - periodInSeconds;
}
std::stringstream query;
if (currentPeriodStamp > newestSourcePeriodStamp) query << "DELETE FROM `" << table << "` WHERE `timestamp` < FROM_UNIXTIME(" << timestamp << ");";
break; m_pMySQLClient->Execute(query.str());
}
samples = 0;
values.clear(); } // namespace DataInterface
periodStamp = currentPeriodStamp; } // namespace DataStorageInterface
}
std::string meanValue = resultSet.String("mean-value");
values.push_back(meanValue);
++samples;
}
if (writeCount > 0)
ExecuteInsertQuery(query);
std::cout << " Write Count: " << writeCount << std::endl << std::endl;
}
void DataCondenser::CleanData()
{
std::cout << "Cleaning: " << std::endl;
CleanData(RRA::Daily);
CleanData(RRA::Weekly);
CleanData(RRA::Monthly);
CleanData(RRA::Yearly);
}
void DataCondenser::CleanData(const RRA::type& rra)
{
std::string table = GetDestinationTable(rra);
int periodInSeconds = static_cast<int>(rra) * 400;
int timestamp = Util::GetTimestamp() + Util::GetUTCOffset() - periodInSeconds;
std::cout << " Cleaning " << table << " (<" << timestamp << ")" << std::endl;
std::stringstream query;
query << "DELETE FROM `" << table << "` WHERE `timestamp` < FROM_UNIXTIME(" << timestamp << ");";
m_pMySQLClient->Execute(query.str());
}
} // namespace DataInterface
} // namespace DataStorageInterface
DataInterface/DataCondenser.h
+52 -52
View File
@@ -1,52 +1,52 @@
#ifndef DATAINTERFACE_DATACONDENSER_H #ifndef DATAINTERFACE_DATACONDENSER_H
#define DATAINTERFACE_DATACONDENSER_H #define DATAINTERFACE_DATACONDENSER_H
#include "DataType.h" #include "DataType.h"
#include "RRA.h" #include "RRA.h"
#include <MySQLClient.h> #include <MySQLClient.h>
#include <string> #include <string>
#include <vector> #include <vector>
namespace DataStorageInterface { namespace DataStorageInterface {
namespace DataInterface { namespace DataInterface {
class DataCondenser class DataCondenser
{ {
public: public:
DataCondenser(MySQL::MySQLClient* pMySQLClient); DataCondenser(MySQL::MySQLClient* pMySQLClient);
void CondenseData(); void CondenseData();
private: private:
int GetPeriodStamp(int timestamp, const RRA::type& rra); int GetPeriodStamp(int timestamp, const RRA::type& rra);
std::vector<int> GetDeviceIDs(); std::vector<int> GetDeviceIDs();
int GetNewestPeriodStamp(const std::string& sourceTable, int deviceId, int dataId); int GetNewestPeriodStamp(const std::string& sourceTable, int deviceId, int dataId);
MySQL::MySQLResultSet GetRawData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp); MySQL::MySQLResultSet GetRawData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp);
MySQL::MySQLResultSet GetReducedData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp); MySQL::MySQLResultSet GetReducedData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp);
int GetMaximumMySQLPacketSize(); int GetMaximumMySQLPacketSize();
DataType::type GetDataType(int deviceId, int dataId); DataType::type GetDataType(int deviceId, int dataId);
std::string GetSourceTable(const RRA::type& rra); std::string GetSourceTable(const RRA::type& rra);
std::string GetDestinationTable(const RRA::type& rra); std::string GetDestinationTable(const RRA::type& rra);
int StringStreamSize(std::stringstream& ss); int StringStreamSize(std::stringstream& ss);
void InitializeInsertQuery(std::stringstream& query, const std::string& table); void InitializeInsertQuery(std::stringstream& query, const std::string& table);
void ExecuteInsertQuery(std::stringstream& query); void ExecuteInsertQuery(std::stringstream& query);
void WriteReducedFloatData(int deviceId, int dataId, const RRA::type& rra); void WriteReducedFloatData(int deviceId, int dataId, const RRA::type& rra);
void WriteReducedStringData(int deviceId, int dataId, const RRA::type& rra); void WriteReducedStringData(int deviceId, int dataId, const RRA::type& rra);
void WriteReducedData(int deviceId, int dataId, const RRA::type& rra); void WriteReducedData(int deviceId, int dataId, const RRA::type& rra);
void CleanData(); void CleanData();
void CleanData(const RRA::type& rra); void CleanData(const RRA::type& rra);
private: private:
MySQL::MySQLClient* m_pMySQLClient; MySQL::MySQLClient* m_pMySQLClient;
}; };
} // namespace DataInterface } // namespace DataInterface
} // namespace DataStorageInterface } // namespace DataStorageInterface
#endif // DATAINTERFACE_DATACONDENSER_H #endif // DATAINTERFACE_DATACONDENSER_H
DataInterface/DataImporter.cpp
+139 -139
View File
@@ -1,139 +1,139 @@
#include "DataImporter.h" #include "DataImporter.h"
#include <DataId.h> #include <DataId.h>
#include <Logging.h> #include <Logging.h>
#include <MySQLClient.h> #include <MySQLClient.h>
#include <SQLiteClient.h> #include <SQLiteClient.h>
#include <StringAlgorithm.h> #include <StringAlgorithm.h>
#include <sstream> #include <sstream>
namespace DataStorageInterface { namespace DataStorageInterface {
namespace DataInterface { namespace DataInterface {
DataImporter::DataImporter(MySQL::MySQLClient* pMySQLClient, SQLite::SQLiteClient* pSQLiteClient) : DataImporter::DataImporter(MySQL::MySQLClient* pMySQLClient, SQLite::SQLiteClient* pSQLiteClient) :
m_pMySQLClient(pMySQLClient), m_pMySQLClient(pMySQLClient),
m_pSQLiteClient(pSQLiteClient) m_pSQLiteClient(pSQLiteClient)
{ {
} }
void DataImporter::ImportData() void DataImporter::ImportData()
{ {
auto tables = GetTables(); auto tables = GetTables();
for (auto& table : tables) for (auto& table : tables)
{ {
auto devices = GetDevices(table); auto devices = GetDevices(table);
for (auto& device : devices) for (auto& device : devices)
ImportData(table, device); ImportData(table, device);
} }
} }
std::vector<std::string> DataImporter::GetTables() std::vector<std::string> DataImporter::GetTables()
{ {
std::stringstream query; std::stringstream query;
query << "SELECT name FROM sqlite_master WHERE type ='table' AND name NOT LIKE 'sqlite_%';"; query << "SELECT name FROM sqlite_master WHERE type ='table' AND name NOT LIKE 'sqlite_%';";
auto resultSet = m_pSQLiteClient->ExecuteQuery(query.str()); auto resultSet = m_pSQLiteClient->ExecuteQuery(query.str());
std::vector<std::string> tables; std::vector<std::string> tables;
while (resultSet.Next()) while (resultSet.Next())
tables.push_back(resultSet.String("name")); tables.push_back(resultSet.String("name"));
return tables; return tables;
} }
std::vector<int> DataImporter::GetDevices(const std::string& table) std::vector<int> DataImporter::GetDevices(const std::string& table)
{ {
std::stringstream query; std::stringstream query;
query << "SELECT DISTINCT device_id FROM " << table << ";"; query << "SELECT DISTINCT device_id FROM " << table << ";";
auto resultSet = m_pSQLiteClient->ExecuteQuery(query.str()); auto resultSet = m_pSQLiteClient->ExecuteQuery(query.str());
std::vector<int> devices; std::vector<int> devices;
while (resultSet.Next()) while (resultSet.Next())
devices.push_back(resultSet.Int("device_id")); devices.push_back(resultSet.Int("device_id"));
return devices; return devices;
} }
void DataImporter::ImportData(const std::string& table, int device) void DataImporter::ImportData(const std::string& table, int device)
{ {
std::stringstream ss; std::stringstream ss;
ss << "Table: " << table << " #" << device; ss << "Table: " << table << " #" << device;
Logging::Log(Logging::Severity::Info, ss.str()); Logging::Log(Logging::Severity::Info, ss.str());
int maxMySQLPacketSize = GetMaximumMySQLPacketSize() - 2; int maxMySQLPacketSize = GetMaximumMySQLPacketSize() - 2;
int maxTimestamp = GetMaximumTimestamp(table, device); int maxTimestamp = GetMaximumTimestamp(table, device);
int dataId = DataStorage::DataId(table).Id(); int dataId = DataStorage::DataId(table).Id();
std::stringstream queryPart; std::stringstream queryPart;
std::stringstream query; std::stringstream query;
query << "SELECT timestamp, value FROM " << table << " WHERE device_id = '" << device << "' AND timestamp > '" << maxTimestamp << "' ORDER BY timestamp ASC;"; query << "SELECT timestamp, value FROM " << table << " WHERE device_id = '" << device << "' AND timestamp > '" << maxTimestamp << "' ORDER BY timestamp ASC;";
auto resultSet = m_pSQLiteClient->ExecuteQuery(query.str()); auto resultSet = m_pSQLiteClient->ExecuteQuery(query.str());
int writeCount = 0; int writeCount = 0;
InitializeInsertQuery(query); InitializeInsertQuery(query);
while (resultSet.Next()) while (resultSet.Next())
{ {
queryPart.str(""); queryPart.str("");
queryPart << "('" << device << "', '" << dataId << "', FROM_UNIXTIME(" << resultSet.Int("timestamp") << "), '" << resultSet.String("value") << "'),"; queryPart << "('" << device << "', '" << dataId << "', FROM_UNIXTIME(" << resultSet.Int("timestamp") << "), '" << resultSet.String("value") << "'),";
++writeCount; ++writeCount;
int packetSize = StringStreamSize(query) + StringStreamSize(queryPart); int packetSize = StringStreamSize(query) + StringStreamSize(queryPart);
if (packetSize > maxMySQLPacketSize) if (packetSize > maxMySQLPacketSize)
{ {
ExecuteInsertQuery(query); ExecuteInsertQuery(query);
writeCount = 0; writeCount = 0;
InitializeInsertQuery(query); InitializeInsertQuery(query);
} }
query << queryPart.str(); query << queryPart.str();
} }
if (writeCount > 0) if (writeCount > 0)
ExecuteInsertQuery(query); ExecuteInsertQuery(query);
} }
int DataImporter::GetMaximumMySQLPacketSize() int DataImporter::GetMaximumMySQLPacketSize()
{ {
std::stringstream query; std::stringstream query;
query << "show variables like 'max_allowed_packet';"; query << "show variables like 'max_allowed_packet';";
MySQL::MySQLResultSet resultSet = m_pMySQLClient->ExecuteQuery(query.str()); MySQL::MySQLResultSet resultSet = m_pMySQLClient->ExecuteQuery(query.str());
if (resultSet.RowsCount() != 1) if (resultSet.RowsCount() != 1)
throw std::runtime_error("Unexpected reply when retrieving max_allowed_packet."); throw std::runtime_error("Unexpected reply when retrieving max_allowed_packet.");
resultSet.Next(); resultSet.Next();
return resultSet.Int("Value"); return resultSet.Int("Value");
} }
int DataImporter::StringStreamSize(std::stringstream& ss) int DataImporter::StringStreamSize(std::stringstream& ss)
{ {
ss.seekp(0, std::ios_base::end); ss.seekp(0, std::ios_base::end);
return ss.tellp(); return ss.tellp();
} }
void DataImporter::InitializeInsertQuery(std::stringstream& query) void DataImporter::InitializeInsertQuery(std::stringstream& query)
{ {
query.str(""); query.str("");
query << "INSERT INTO `datalog` (`device_id`, `data_id`, `timestamp`, `value`) VALUES "; query << "INSERT INTO `datalog` (`device_id`, `data_id`, `timestamp`, `value`) VALUES ";
} }
void DataImporter::ExecuteInsertQuery(std::stringstream& query) void DataImporter::ExecuteInsertQuery(std::stringstream& query)
{ {
query.seekp(-1, std::ios_base::end); query.seekp(-1, std::ios_base::end);
query << ";"; query << ";";
m_pMySQLClient->Execute(query.str()); m_pMySQLClient->Execute(query.str());
} }
int DataImporter::GetMaximumTimestamp(const std::string& table, int device) int DataImporter::GetMaximumTimestamp(const std::string& table, int device)
{ {
std::stringstream query; std::stringstream query;
query << "SELECT MAX(UNIX_TIMESTAMP(`timestamp`)) AS timestamp FROM `datalog` WHERE `device_id` = '" << device <<"' AND `data_id` = '" << DataStorage::DataId(table).Id() << "';"; query << "SELECT MAX(UNIX_TIMESTAMP(`timestamp`)) AS timestamp FROM `datalog` WHERE `device_id` = '" << device <<"' AND `data_id` = '" << DataStorage::DataId(table).Id() << "';";
auto resultSet = m_pMySQLClient->ExecuteQuery(query.str()); auto resultSet = m_pMySQLClient->ExecuteQuery(query.str());
if (resultSet.Next()) if (resultSet.Next())
return resultSet.Int("timestamp"); return resultSet.Int("timestamp");
return 0; return 0;
} }
} // namespace DataInterface } // namespace DataInterface
} // namespace DataStorageInterface } // namespace DataStorageInterface
DataInterface/DataImporter.h
+49 -49
View File
@@ -1,49 +1,49 @@
#ifndef DATAINTERFACE_DATAIMPORTER_H #ifndef DATAINTERFACE_DATAIMPORTER_H
#define DATAINTERFACE_DATAIMPORTER_H #define DATAINTERFACE_DATAIMPORTER_H
#include <string> #include <string>
#include <vector> #include <vector>
namespace MySQL { namespace MySQL {
class MySQLClient; class MySQLClient;
} // namespace MySQL } // namespace MySQL
namespace SQLite { namespace SQLite {
class SQLiteClient; class SQLiteClient;
} // namespace SQLite } // namespace SQLite
namespace DataStorageInterface { namespace DataStorageInterface {
namespace DataInterface { namespace DataInterface {
class DataImporter class DataImporter
{ {
public: public:
DataImporter(MySQL::MySQLClient* pMySQLClient, SQLite::SQLiteClient* pSQLiteClient); DataImporter(MySQL::MySQLClient* pMySQLClient, SQLite::SQLiteClient* pSQLiteClient);
void ImportData(); void ImportData();
private: private:
std::vector<std::string> GetTables(); std::vector<std::string> GetTables();
std::vector<int> GetDevices(const std::string& table); std::vector<int> GetDevices(const std::string& table);
void ImportData(const std::string& table, int device); void ImportData(const std::string& table, int device);
int GetMaximumMySQLPacketSize(); int GetMaximumMySQLPacketSize();
int StringStreamSize(std::stringstream& ss); int StringStreamSize(std::stringstream& ss);
void InitializeInsertQuery(std::stringstream& query); void InitializeInsertQuery(std::stringstream& query);
void ExecuteInsertQuery(std::stringstream& query); void ExecuteInsertQuery(std::stringstream& query);
int GetMaximumTimestamp(const std::string& table, int device); int GetMaximumTimestamp(const std::string& table, int device);
private: private:
MySQL::MySQLClient* m_pMySQLClient; MySQL::MySQLClient* m_pMySQLClient;
SQLite::SQLiteClient* m_pSQLiteClient; SQLite::SQLiteClient* m_pSQLiteClient;
}; };
} // namespace DataInterface } // namespace DataInterface
} // namespace DataStorageInterface } // namespace DataStorageInterface
#endif // DATAINTERFACE_DATAIMPORTER_H #endif // DATAINTERFACE_DATAIMPORTER_H
DataInterface/GraphClient.cpp
+117 -117
View File
@@ -1,117 +1,117 @@
#include "GraphClient.h" #include "GraphClient.h"
#include "Util/Util.h" #include <DateTime.h>
#include <MySQLClient.h> #include <MySQLClient.h>
#include <StringAlgorithm.h> #include <StringAlgorithm.h>
#include <sstream> #include <sstream>
namespace DataStorageInterface { namespace DataStorageInterface {
namespace DataInterface { namespace DataInterface {
GraphClient::GraphClient(MySQL::MySQLClient* pMySQLClient) : GraphClient::GraphClient(MySQL::MySQLClient* pMySQLClient) :
m_pMySQLClient(pMySQLClient) m_pMySQLClient(pMySQLClient)
{ {
} }
nlohmann::json GraphClient::GetGraphHeader(const std::vector<int>& dataIds, DataStorage::Timespan::type timespan) nlohmann::json GraphClient::GetGraphHeader(const std::vector<int>& dataIds, DataStorage::Timespan::type timespan)
{ {
int currentTimestamp = Util::GetTimestamp() + Util::GetUTCOffset(); int currentTimestamp = DateTime::GetTimestamp() + DateTime::GetUTCOffset();
return GetGraphHeader(timespan, currentTimestamp); return GetGraphHeader(timespan, currentTimestamp);
} }
nlohmann::json GraphClient::GetGraphData(int deviceId, const std::vector<int>& dataIds, DataStorage::Timespan::type timespan) nlohmann::json GraphClient::GetGraphData(int deviceId, const std::vector<int>& dataIds, DataStorage::Timespan::type timespan)
{ {
int currentTimestamp = Util::GetTimestamp() + Util::GetUTCOffset(); int currentTimestamp = DateTime::GetTimestamp() + DateTime::GetUTCOffset();
nlohmann::json json = GetGraphHeader(timespan, currentTimestamp); nlohmann::json json = GetGraphHeader(timespan, currentTimestamp);
std::string table; std::string table;
switch (timespan) switch (timespan)
{ {
case DataStorage::Timespan::Day: case DataStorage::Timespan::Day:
table = "datalog"; // "datalog-daily" table = "datalog"; // "datalog-daily"
break; break;
case DataStorage::Timespan::Week: case DataStorage::Timespan::Week:
table = "datalog-weekly"; table = "datalog-weekly";
break; break;
case DataStorage::Timespan::Month: case DataStorage::Timespan::Month:
table = "datalog-monthly"; table = "datalog-monthly";
break; break;
case DataStorage::Timespan::Year: case DataStorage::Timespan::Year:
table = "datalog-yearly"; table = "datalog-yearly";
break; break;
default: default:
case DataStorage::Timespan::Unknown: case DataStorage::Timespan::Unknown:
return nlohmann::json(); return nlohmann::json();
} }
nlohmann::json rootData = nlohmann::json::array(); nlohmann::json rootData = nlohmann::json::array();
std::stringstream query; std::stringstream query;
int startTimestamp = Util::GetTimestamp() - timespan; int startTimestamp = DateTime::GetTimestamp() - timespan;
for (auto dataId : dataIds) for (auto dataId : dataIds)
{ {
nlohmann::json data; nlohmann::json data;
nlohmann::json dataArray; nlohmann::json dataArray;
query.str(""); query.str("");
if (StringAlgorithm::iequals(table, "datalog")) if (StringAlgorithm::iequals(table, "datalog"))
query << "SELECT UNIX_TIMESTAMP(`d`.`timestamp`) AS timestamp, `d`.`value` AS 'min-value', `d`.`value` AS 'mean-value', `d`.`value` AS 'max-value' "; query << "SELECT UNIX_TIMESTAMP(`d`.`timestamp`) AS timestamp, `d`.`value` AS 'min-value', `d`.`value` AS 'mean-value', `d`.`value` AS 'max-value' ";
else else
query << "SELECT UNIX_TIMESTAMP(`d`.`timestamp`) AS timestamp, `d`.`min-value`, `d`.`mean-value`, `d`.`max-value` "; query << "SELECT UNIX_TIMESTAMP(`d`.`timestamp`) AS timestamp, `d`.`min-value`, `d`.`mean-value`, `d`.`max-value` ";
query << "FROM `" << table << "` AS `d` "; query << "FROM `" << table << "` AS `d` ";
query << "WHERE `d`.`device_id` = '" << deviceId << "' "; query << "WHERE `d`.`device_id` = '" << deviceId << "' ";
query << "AND `d`.`data_id` = '" << dataId << "' "; query << "AND `d`.`data_id` = '" << dataId << "' ";
query << "AND UNIX_TIMESTAMP(`d`.`timestamp`) > '" << startTimestamp << "';"; query << "AND UNIX_TIMESTAMP(`d`.`timestamp`) > '" << startTimestamp << "';";
auto resultSet = m_pMySQLClient->ExecuteQuery(query.str()); auto resultSet = m_pMySQLClient->ExecuteQuery(query.str());
int offset = Util::GetUTCOffset(); int offset = DateTime::GetUTCOffset();
int timestamp; int timestamp;
double minValue, meanValue, maxValue; double minValue, meanValue, maxValue;
if (resultSet.RowsCount() > 0) if (resultSet.RowsCount() > 0)
{ {
while (resultSet.Next()) while (resultSet.Next())
{ {
timestamp = resultSet.Int("timestamp") + offset; timestamp = resultSet.Int("timestamp") + offset;
minValue = resultSet.Double("min-value"); minValue = resultSet.Double("min-value");
meanValue = resultSet.Double("mean-value"); meanValue = resultSet.Double("mean-value");
maxValue = resultSet.Double("max-value"); maxValue = resultSet.Double("max-value");
nlohmann::json entry = nlohmann::json::array(); nlohmann::json entry = nlohmann::json::array();
entry.push_back(timestamp); entry.push_back(timestamp);
std::stringstream entryValue; std::stringstream entryValue;
if (StringAlgorithm::iequals(table, "datalog")) if (StringAlgorithm::iequals(table, "datalog"))
entryValue << std::fixed << meanValue; entryValue << std::fixed << meanValue;
else else
entryValue << std::fixed << minValue << ", " << std::fixed << maxValue; entryValue << std::fixed << minValue << ", " << std::fixed << maxValue;
entry.push_back(entryValue.str()); entry.push_back(entryValue.str());
dataArray.push_back(entry); dataArray.push_back(entry);
} }
data["data"] = dataArray; data["data"] = dataArray;
rootData.push_back(data); rootData.push_back(data);
} }
} }
json["data"] = rootData; json["data"] = rootData;
return json; return json;
} }
nlohmann::json GraphClient::GetGraphHeader(DataStorage::Timespan::type timespan, std::time_t currentTimestamp) nlohmann::json GraphClient::GetGraphHeader(DataStorage::Timespan::type timespan, std::time_t currentTimestamp)
{ {
int startTimestamp = currentTimestamp - timespan; int startTimestamp = currentTimestamp - timespan;
nlohmann::json json; nlohmann::json json;
json["timespan"] = DataStorage::Conversions::Timespan(timespan); json["timespan"] = DataStorage::Conversions::Timespan(timespan);
json["start"] = startTimestamp; json["start"] = startTimestamp;
json["end"] = currentTimestamp; json["end"] = currentTimestamp;
return json; return json;
} }
} // namespace DataInterface } // namespace DataInterface
} // namespace DataStorageInterface } // namespace DataStorageInterface
DataInterface/GraphClient.h
+37 -37
View File
@@ -1,37 +1,37 @@
#ifndef DATAINTERFACE_GRAPHCLIENT_H #ifndef DATAINTERFACE_GRAPHCLIENT_H
#define DATAINTERFACE_GRAPHCLIENT_H #define DATAINTERFACE_GRAPHCLIENT_H
#include <Timespan.h> #include <Timespan.h>
#include <json.hpp> #include <json.hpp>
#include <ctime> #include <ctime>
#include <vector> #include <vector>
namespace MySQL { namespace MySQL {
class MySQLClient; class MySQLClient;
} // namespace MySQL } // namespace MySQL
namespace DataStorageInterface { namespace DataStorageInterface {
namespace DataInterface { namespace DataInterface {
class GraphClient class GraphClient
{ {
public: public:
GraphClient(MySQL::MySQLClient* pMySQLClient); GraphClient(MySQL::MySQLClient* pMySQLClient);
nlohmann::json GetGraphHeader(const std::vector<int>& dataIds, DataStorage::Timespan::type timespan); nlohmann::json GetGraphHeader(const std::vector<int>& dataIds, DataStorage::Timespan::type timespan);
nlohmann::json GetGraphData(int deviceId, const std::vector<int>& dataIds, DataStorage::Timespan::type timespan); nlohmann::json GetGraphData(int deviceId, const std::vector<int>& dataIds, DataStorage::Timespan::type timespan);
private: private:
nlohmann::json GetGraphHeader(DataStorage::Timespan::type timespan, std::time_t currentTimestamp); nlohmann::json GetGraphHeader(DataStorage::Timespan::type timespan, std::time_t currentTimestamp);
private: private:
MySQL::MySQLClient* m_pMySQLClient; MySQL::MySQLClient* m_pMySQLClient;
}; };
} // namespace DataInterface } // namespace DataInterface
} // namespace DataStorageInterface } // namespace DataStorageInterface
#endif // DATAINTERFACE_GRAPHCLIENT_H #endif // DATAINTERFACE_GRAPHCLIENT_H
Libraries/DateTime
Symlink
+1
View File
@@ -0,0 +1 @@
../../Libraries/DateTime
Makefile.conf
+8 -2
View File
@@ -14,7 +14,7 @@ LFLAGS += -lDataStorage
LFLAGS += -L$(ROOTPATH)/Libraries/DataStorage/lib/$(ARCH) LFLAGS += -L$(ROOTPATH)/Libraries/DataStorage/lib/$(ARCH)
CFLAGS += -I$(ROOTPATH)/Libraries/DataStorage/include CFLAGS += -I$(ROOTPATH)/Libraries/DataStorage/include
LFLAGS += -lHttp LFLAGS += -lHttp -lcrypto -lcurl -lssl -lz
LFLAGS += -L$(ROOTPATH)/Libraries/Http/lib/$(ARCH) LFLAGS += -L$(ROOTPATH)/Libraries/Http/lib/$(ARCH)
CFLAGS += -I$(ROOTPATH)/Libraries/Http/include CFLAGS += -I$(ROOTPATH)/Libraries/Http/include
@@ -26,10 +26,16 @@ LFLAGS += -lUtilities
LFLAGS += -L$(ROOTPATH)/Libraries/Utilities/lib/$(ARCH) LFLAGS += -L$(ROOTPATH)/Libraries/Utilities/lib/$(ARCH)
CFLAGS += -I$(ROOTPATH)/Libraries/Utilities/include CFLAGS += -I$(ROOTPATH)/Libraries/Utilities/include
LFLAGS += -lDateTime
LFLAGS += -L$(ROOTPATH)/Libraries/DateTime/lib/$(ARCH)
CFLAGS += -I$(ROOTPATH)/Libraries/DateTime/include
CFLAGS += -I$(ROOTPATH)/Libraries/clipp/include CFLAGS += -I$(ROOTPATH)/Libraries/clipp/include
CFLAGS += -I$(ROOTPATH)/Libraries/inih CFLAGS += -I$(ROOTPATH)/Libraries/inih
CFLAGS += -I$(ROOTPATH)/Libraries/json/include/nlohmann -I$(ROOTPATH)/Libraries/json/include CFLAGS += -I$(ROOTPATH)/Libraries/json/include/nlohmann -I$(ROOTPATH)/Libraries/json/include
LFLAGS += -pthread -lm LFLAGS += -pthread
CFLAGS += -I$(ROOTPATH) -I$(ROOTPATH)/Libraries CFLAGS += -I$(ROOTPATH) -I$(ROOTPATH)/Libraries
DEBUGDIR := .debug DEBUGDIR := .debug
Util/Makefile
-1
View File
@@ -1 +0,0 @@
../Makefile
Util/Util.cpp
-23
View File
@@ -1,23 +0,0 @@
#include "Util.h"
#include <algorithm>
#include <ctime>
namespace DataStorageInterface {
namespace Util {
int GetTimestamp()
{
return std::time(nullptr);
}
int GetUTCOffset()
{
std::time_t current_time;
std::time(&current_time);
struct std::tm *timeinfo = std::localtime(&current_time);
return timeinfo->tm_gmtoff;
}
} // namespace Util
} // namespace DataStorageInterface
Util/Util.h
-16
View File
@@ -1,16 +0,0 @@
#ifndef UTIL_UTIL_H
#define UTIL_UTIL_H
#include <string>
namespace DataStorageInterface {
namespace Util {
int GetTimestamp();
int GetUTCOffset();
} // namespace Util
} // namespace DataStorageInterface
#endif // UTIL_UTIL_H