Initial commit
This commit is contained in:
@@ -0,0 +1,406 @@
|
||||
#include "DataCondenser.h"
|
||||
#include "Util/Util.h"
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <sstream>
|
||||
|
||||
|
||||
namespace DataStorageInterface {
|
||||
namespace DataInterface {
|
||||
|
||||
DataCondenser::DataCondenser(MySQL::MySQLClient* pMySQLClient) :
|
||||
m_pMySQLClient(pMySQLClient)
|
||||
{
|
||||
}
|
||||
|
||||
void DataCondenser::CondenseData()
|
||||
{
|
||||
auto deviceIDs = GetDeviceIDs();
|
||||
|
||||
for (int dataId = 0; dataId < 3; ++dataId)
|
||||
{
|
||||
std::cout << "DataID: " << dataId << std::endl;
|
||||
|
||||
//std::cout << "Daily" << std::endl;
|
||||
//for (auto& deviceId : deviceIDs)
|
||||
// WriteReducedData(deviceId, dataId, RRA::Daily);
|
||||
std::cout << "Weekly" << std::endl;
|
||||
for (auto& deviceId : deviceIDs)
|
||||
WriteReducedData(deviceId, dataId, RRA::Weekly);
|
||||
std::cout << "Monthly" << std::endl;
|
||||
for (auto& deviceId : deviceIDs)
|
||||
WriteReducedData(deviceId, dataId, RRA::Monthly);
|
||||
std::cout << "Yearly" << std::endl;
|
||||
for (auto& deviceId : deviceIDs)
|
||||
WriteReducedData(deviceId, dataId, RRA::Yearly);
|
||||
}
|
||||
|
||||
CleanData();
|
||||
}
|
||||
|
||||
int DataCondenser::GetPeriodStamp(int timestamp, const RRA::type& rra)
|
||||
{
|
||||
int modulo = timestamp % static_cast<int>(rra);
|
||||
return timestamp - modulo;
|
||||
}
|
||||
|
||||
std::vector<int> DataCondenser::GetDeviceIDs()
|
||||
{
|
||||
std::stringstream query;
|
||||
query << "SELECT DISTINCT `device_id` FROM `datalog` ORDER BY `device_id`;";
|
||||
auto resultSet = m_pMySQLClient->ExecuteQuery(query.str());
|
||||
|
||||
std::vector<int> returnValue;
|
||||
while (resultSet.Next())
|
||||
returnValue.push_back(resultSet.Int("device_id"));
|
||||
|
||||
return returnValue;
|
||||
}
|
||||
|
||||
int DataCondenser::GetNewestPeriodStamp(const std::string& sourceTable, int deviceId, int dataId)
|
||||
{
|
||||
std::stringstream query;
|
||||
query << "SELECT MAX(UNIX_TIMESTAMP(`timestamp`)) AS timestamp FROM `" << sourceTable << "` WHERE `device_id` = '" << deviceId <<"' AND `data_id` = '" << dataId << "';";
|
||||
auto resultSet = m_pMySQLClient->ExecuteQuery(query.str());
|
||||
if (resultSet.Next())
|
||||
return resultSet.Int("timestamp");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
MySQL::MySQLResultSet DataCondenser::GetRawData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp)
|
||||
{
|
||||
std::stringstream query;
|
||||
query << "SELECT UNIX_TIMESTAMP(`timestamp`) AS timestamp, `value` AS `min-value`, `value` AS `mean-value`, `value` AS `max-value` FROM `" << sourceTable << "` WHERE `device_id` = '" << deviceId <<"' AND `data_id` = '" << dataId << "' ";
|
||||
if (newestPeriodStamp == -1)
|
||||
query << "LIMIT 1;";
|
||||
else
|
||||
query << "AND `timestamp` > FROM_UNIXTIME(" << newestPeriodStamp << ") ORDER BY `timestamp` ASC;";
|
||||
|
||||
return m_pMySQLClient->ExecuteQuery(query.str());
|
||||
}
|
||||
|
||||
MySQL::MySQLResultSet DataCondenser::GetReducedData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp)
|
||||
{
|
||||
std::stringstream query;
|
||||
query << "SELECT UNIX_TIMESTAMP(`timestamp`) AS timestamp, `min-value`, `mean-value`, `max-value` FROM `" << sourceTable << "` WHERE `device_id` = '" << deviceId <<"' AND `data_id` = '" << dataId << "' ";
|
||||
if (newestPeriodStamp == -1)
|
||||
query << "LIMIT 1;";
|
||||
else
|
||||
query << "AND `timestamp` > FROM_UNIXTIME(" << newestPeriodStamp << ") ORDER BY `timestamp` ASC;";
|
||||
|
||||
return m_pMySQLClient->ExecuteQuery(query.str());
|
||||
}
|
||||
|
||||
int DataCondenser::GetMaximumMySQLPacketSize()
|
||||
{
|
||||
std::stringstream query;
|
||||
query << "show variables like 'max_allowed_packet';";
|
||||
MySQL::MySQLResultSet resultSet = m_pMySQLClient->ExecuteQuery(query.str());
|
||||
|
||||
if (resultSet.RowsCount() != 1)
|
||||
throw std::runtime_error("Unexpected reply when retrieving max_allowed_packet.");
|
||||
|
||||
resultSet.Next();
|
||||
return resultSet.Int("Value");
|
||||
}
|
||||
|
||||
DataType::type DataCondenser::GetDataType(int deviceId, int dataId)
|
||||
{
|
||||
auto resultSet = GetRawData("datalog", deviceId, dataId, -1);
|
||||
if (resultSet.Next())
|
||||
return DataInterface::GetDataType(resultSet.String("mean-value"));
|
||||
|
||||
return DataType::None;
|
||||
}
|
||||
|
||||
std::string DataCondenser::GetSourceTable(const RRA::type& rra)
|
||||
{
|
||||
switch (rra)
|
||||
{
|
||||
case RRA::Daily:
|
||||
return "datalog";
|
||||
case RRA::Weekly:
|
||||
return "datalog";
|
||||
case RRA::Monthly:
|
||||
return "datalog-weekly";
|
||||
case RRA::Yearly:
|
||||
return "datalog-monthly";
|
||||
default:
|
||||
case RRA::Raw:
|
||||
return std::string();
|
||||
}
|
||||
}
|
||||
|
||||
std::string DataCondenser::GetDestinationTable(const RRA::type& rra)
|
||||
{
|
||||
switch (rra)
|
||||
{
|
||||
case RRA::Daily:
|
||||
return "datalog-daily";
|
||||
case RRA::Weekly:
|
||||
return "datalog-weekly";
|
||||
case RRA::Monthly:
|
||||
return "datalog-monthly";
|
||||
case RRA::Yearly:
|
||||
return "datalog-yearly";
|
||||
default:
|
||||
case RRA::Raw:
|
||||
return std::string();
|
||||
}
|
||||
}
|
||||
|
||||
int DataCondenser::StringStreamSize(std::stringstream& ss)
|
||||
{
|
||||
ss.seekp(0, std::ios_base::end);
|
||||
return ss.tellp();
|
||||
}
|
||||
|
||||
void DataCondenser::InitializeInsertQuery(std::stringstream& query, const std::string& table)
|
||||
{
|
||||
query.str("");
|
||||
query << "INSERT INTO `" << table << "` (`device_id`, `data_id`, `timestamp`, `min-value`, `mean-value`, `max-value`) VALUES ";
|
||||
}
|
||||
|
||||
void DataCondenser::ExecuteInsertQuery(std::stringstream& query)
|
||||
{
|
||||
query.seekp(-1, std::ios_base::end);
|
||||
query << ";";
|
||||
m_pMySQLClient->Execute(query.str());
|
||||
}
|
||||
|
||||
void DataCondenser::WriteReducedData(int deviceId, int dataId, const RRA::type& rra)
|
||||
{
|
||||
DataType::type dataType = GetDataType(deviceId, dataId);
|
||||
|
||||
if (dataType == DataType::None)
|
||||
std::cout << " DataType: None" << std::endl;
|
||||
else if (dataType == DataType::Float)
|
||||
std::cout << " DataType: Float" << std::endl;
|
||||
else if (dataType == DataType::String)
|
||||
std::cout << " DataType: String" << std::endl;
|
||||
|
||||
switch (dataType)
|
||||
{
|
||||
case DataType::Float:
|
||||
WriteReducedFloatData(deviceId, dataId, rra);
|
||||
break;
|
||||
case DataType::String:
|
||||
WriteReducedStringData(deviceId, dataId, rra);
|
||||
break;
|
||||
default:
|
||||
case DataType::None:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void DataCondenser::WriteReducedFloatData(int deviceId, int dataId, const RRA::type& rra)
|
||||
{
|
||||
int maxMySQLPacketSize = GetMaximumMySQLPacketSize() - 2;
|
||||
|
||||
std::string sourceTable = GetSourceTable(rra);
|
||||
std::string destinationTable = GetDestinationTable(rra);
|
||||
|
||||
int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId);
|
||||
int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId);
|
||||
|
||||
std::cout << " DeviceID: " << deviceId << std::endl;
|
||||
std::cout << " DataID: " << dataId << std::endl;
|
||||
std::cout << " Newest Src : " << newestSourcePeriodStamp << std::endl;
|
||||
std::cout << " Newest Dest: " << newestDestinationPeriodStamp << std::endl;
|
||||
|
||||
std::stringstream query;
|
||||
MySQL::MySQLResultSet resultSet;
|
||||
if (sourceTable == "datalog")
|
||||
resultSet = GetRawData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
|
||||
else
|
||||
resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
|
||||
|
||||
std::cout << " Source Count: " << resultSet.RowsCount() << std::endl;
|
||||
|
||||
int samples = 0;
|
||||
float min = std::numeric_limits<float>::max();
|
||||
float mean = 0;
|
||||
float max = std::numeric_limits<float>::min();
|
||||
|
||||
int writeCount = 0;
|
||||
|
||||
InitializeInsertQuery(query, destinationTable);
|
||||
|
||||
std::stringstream queryPart;
|
||||
|
||||
int periodStamp = 0;
|
||||
while (resultSet.Next())
|
||||
{
|
||||
int currentPeriodStamp = GetPeriodStamp(resultSet.Int("timestamp"), rra);
|
||||
if (currentPeriodStamp != periodStamp)
|
||||
{
|
||||
if (samples > 0)
|
||||
{
|
||||
mean = mean / samples;
|
||||
|
||||
queryPart.str("");
|
||||
queryPart << "('" << deviceId << "', '" << dataId << "', FROM_UNIXTIME(" << currentPeriodStamp << "), '" << min << "', '" << mean << "', '" << max << "'),";
|
||||
++writeCount;
|
||||
|
||||
int packetSize = StringStreamSize(query) + StringStreamSize(queryPart);
|
||||
if (packetSize > maxMySQLPacketSize)
|
||||
{
|
||||
ExecuteInsertQuery(query);
|
||||
InitializeInsertQuery(query, destinationTable);
|
||||
std::cout << " Write Count: " << writeCount << std::endl;
|
||||
writeCount = 0;
|
||||
}
|
||||
|
||||
query << queryPart.str();
|
||||
}
|
||||
|
||||
if (currentPeriodStamp > newestSourcePeriodStamp)
|
||||
break;
|
||||
|
||||
samples = 0;
|
||||
min = std::numeric_limits<float>::max();
|
||||
mean = 0;
|
||||
max = std::numeric_limits<float>::min();
|
||||
periodStamp = currentPeriodStamp;
|
||||
}
|
||||
|
||||
float minValue, meanValue, maxValue;
|
||||
minValue = static_cast<float>(resultSet.Double("min-value"));
|
||||
meanValue = static_cast<float>(resultSet.Double("mean-value"));
|
||||
maxValue = static_cast<float>(resultSet.Double("max-value"));
|
||||
|
||||
if (minValue < min)
|
||||
min = minValue;
|
||||
if (maxValue > max)
|
||||
max = maxValue;
|
||||
mean += meanValue;
|
||||
++samples;
|
||||
}
|
||||
|
||||
if (writeCount > 0)
|
||||
ExecuteInsertQuery(query);
|
||||
|
||||
std::cout << " Write Count: " << writeCount << std::endl << std::endl;
|
||||
}
|
||||
|
||||
void DataCondenser::WriteReducedStringData(int deviceId, int dataId, const RRA::type& rra)
|
||||
{
|
||||
int maxMySQLPacketSize = GetMaximumMySQLPacketSize() - 2;
|
||||
|
||||
std::string sourceTable = GetSourceTable(rra);
|
||||
std::string destinationTable = GetDestinationTable(rra);
|
||||
|
||||
int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId);
|
||||
int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId);
|
||||
|
||||
std::cout << " DeviceID: " << deviceId << std::endl;
|
||||
std::cout << " DataID: " << dataId << std::endl;
|
||||
std::cout << " Newest Src : " << newestSourcePeriodStamp << std::endl;
|
||||
std::cout << " Newest Dest: " << newestDestinationPeriodStamp << std::endl;
|
||||
|
||||
std::stringstream query;
|
||||
MySQL::MySQLResultSet resultSet;
|
||||
if (sourceTable == "datalog")
|
||||
resultSet = GetRawData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
|
||||
else
|
||||
resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
|
||||
|
||||
std::cout << " Source Count: " << resultSet.RowsCount() << std::endl;
|
||||
|
||||
int samples = 0;
|
||||
std::vector<std::string> values;
|
||||
|
||||
int writeCount = 0;
|
||||
|
||||
InitializeInsertQuery(query, destinationTable);
|
||||
|
||||
std::stringstream queryPart;
|
||||
|
||||
int periodStamp = 0;
|
||||
while (resultSet.Next())
|
||||
{
|
||||
int currentPeriodStamp = GetPeriodStamp(resultSet.Int("timestamp"), rra);
|
||||
if (currentPeriodStamp != periodStamp)
|
||||
{
|
||||
if (samples > 0)
|
||||
{
|
||||
std::map<std::string, int> map;
|
||||
for (auto& value : values)
|
||||
{
|
||||
auto iterator = map.find(value);
|
||||
if (iterator == map.end())
|
||||
map.insert(std::pair<std::string, int>(value, 1));
|
||||
else
|
||||
map[value] += 1;
|
||||
}
|
||||
|
||||
auto iterator = map.begin();
|
||||
for (std::map<std::string, int>::iterator it = map.begin(); it != map.end(); ++it)
|
||||
{
|
||||
if (it->second > iterator->second)
|
||||
iterator = it;
|
||||
}
|
||||
|
||||
queryPart.str("");
|
||||
queryPart << "('" << deviceId << "', '" << dataId << "', FROM_UNIXTIME(" << currentPeriodStamp << "), '', '" << iterator->first << "', ''),";
|
||||
|
||||
int packetSize = StringStreamSize(query) + StringStreamSize(queryPart);
|
||||
if (packetSize > maxMySQLPacketSize)
|
||||
{
|
||||
ExecuteInsertQuery(query);
|
||||
InitializeInsertQuery(query, destinationTable);
|
||||
std::cout << " Write Count: " << writeCount << std::endl;
|
||||
writeCount = 0;
|
||||
}
|
||||
|
||||
query << queryPart.str();
|
||||
++writeCount;
|
||||
}
|
||||
|
||||
if (currentPeriodStamp > newestSourcePeriodStamp)
|
||||
break;
|
||||
|
||||
samples = 0;
|
||||
values.clear();
|
||||
periodStamp = currentPeriodStamp;
|
||||
}
|
||||
|
||||
std::string meanValue = resultSet.String("mean-value");
|
||||
values.push_back(meanValue);
|
||||
++samples;
|
||||
}
|
||||
|
||||
if (writeCount > 0)
|
||||
ExecuteInsertQuery(query);
|
||||
|
||||
std::cout << " Write Count: " << writeCount << std::endl << std::endl;
|
||||
}
|
||||
|
||||
void DataCondenser::CleanData()
|
||||
{
|
||||
std::cout << "Cleaning: " << std::endl;
|
||||
|
||||
CleanData(RRA::Daily);
|
||||
CleanData(RRA::Weekly);
|
||||
CleanData(RRA::Monthly);
|
||||
CleanData(RRA::Yearly);
|
||||
}
|
||||
|
||||
void DataCondenser::CleanData(const RRA::type& rra)
|
||||
{
|
||||
std::string table = GetDestinationTable(rra);
|
||||
int periodInSeconds = static_cast<int>(rra) * 400;
|
||||
int timestamp = Util::GetTimestamp() + Util::GetUTCOffset() - periodInSeconds;
|
||||
|
||||
std::cout << " Cleaning " << table << " (<" << timestamp << ")" << std::endl;
|
||||
|
||||
std::stringstream query;
|
||||
query << "DELETE FROM `" << table << "` WHERE `timestamp` < FROM_UNIXTIME(" << timestamp << ");";
|
||||
m_pMySQLClient->Execute(query.str());
|
||||
}
|
||||
|
||||
} // namespace DataInterface
|
||||
} // namespace DataStorageInterface
|
||||
@@ -0,0 +1,52 @@
|
||||
#ifndef DATAINTERFACE_DATACONDENSER_H
|
||||
#define DATAINTERFACE_DATACONDENSER_H
|
||||
|
||||
#include "DataType.h"
|
||||
#include "RRA.h"
|
||||
#include <MySQLClient.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace DataStorageInterface {
|
||||
namespace DataInterface {
|
||||
|
||||
class DataCondenser
|
||||
{
|
||||
public:
|
||||
DataCondenser(MySQL::MySQLClient* pMySQLClient);
|
||||
|
||||
void CondenseData();
|
||||
|
||||
private:
|
||||
int GetPeriodStamp(int timestamp, const RRA::type& rra);
|
||||
std::vector<int> GetDeviceIDs();
|
||||
int GetNewestPeriodStamp(const std::string& sourceTable, int deviceId, int dataId);
|
||||
MySQL::MySQLResultSet GetRawData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp);
|
||||
MySQL::MySQLResultSet GetReducedData(const std::string& sourceTable, int deviceId, int dataId, int newestPeriodStamp);
|
||||
|
||||
int GetMaximumMySQLPacketSize();
|
||||
|
||||
DataType::type GetDataType(int deviceId, int dataId);
|
||||
std::string GetSourceTable(const RRA::type& rra);
|
||||
std::string GetDestinationTable(const RRA::type& rra);
|
||||
int StringStreamSize(std::stringstream& ss);
|
||||
|
||||
void InitializeInsertQuery(std::stringstream& query, const std::string& table);
|
||||
void ExecuteInsertQuery(std::stringstream& query);
|
||||
|
||||
void WriteReducedFloatData(int deviceId, int dataId, const RRA::type& rra);
|
||||
void WriteReducedStringData(int deviceId, int dataId, const RRA::type& rra);
|
||||
void WriteReducedData(int deviceId, int dataId, const RRA::type& rra);
|
||||
|
||||
void CleanData();
|
||||
void CleanData(const RRA::type& rra);
|
||||
|
||||
private:
|
||||
MySQL::MySQLClient* m_pMySQLClient;
|
||||
};
|
||||
|
||||
} // namespace DataInterface
|
||||
} // namespace DataStorageInterface
|
||||
|
||||
#endif // DATAINTERFACE_DATACONDENSER_H
|
||||
@@ -0,0 +1,139 @@
|
||||
#include "DataImporter.h"
|
||||
#include <DataId.h>
|
||||
#include <Logging.h>
|
||||
#include <MySQLClient.h>
|
||||
#include <SQLiteClient.h>
|
||||
#include <StringAlgorithm.h>
|
||||
#include <sstream>
|
||||
|
||||
|
||||
namespace DataStorageInterface {
|
||||
namespace DataInterface {
|
||||
|
||||
DataImporter::DataImporter(MySQL::MySQLClient* pMySQLClient, SQLite::SQLiteClient* pSQLiteClient) :
|
||||
m_pMySQLClient(pMySQLClient),
|
||||
m_pSQLiteClient(pSQLiteClient)
|
||||
{
|
||||
}
|
||||
|
||||
void DataImporter::ImportData()
|
||||
{
|
||||
auto tables = GetTables();
|
||||
for (auto& table : tables)
|
||||
{
|
||||
auto devices = GetDevices(table);
|
||||
for (auto& device : devices)
|
||||
ImportData(table, device);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string> DataImporter::GetTables()
|
||||
{
|
||||
std::stringstream query;
|
||||
query << "SELECT name FROM sqlite_master WHERE type ='table' AND name NOT LIKE 'sqlite_%';";
|
||||
auto resultSet = m_pSQLiteClient->ExecuteQuery(query.str());
|
||||
|
||||
std::vector<std::string> tables;
|
||||
while (resultSet.Next())
|
||||
tables.push_back(resultSet.String("name"));
|
||||
|
||||
return tables;
|
||||
}
|
||||
|
||||
std::vector<int> DataImporter::GetDevices(const std::string& table)
|
||||
{
|
||||
std::stringstream query;
|
||||
query << "SELECT DISTINCT device_id FROM " << table << ";";
|
||||
auto resultSet = m_pSQLiteClient->ExecuteQuery(query.str());
|
||||
|
||||
std::vector<int> devices;
|
||||
while (resultSet.Next())
|
||||
devices.push_back(resultSet.Int("device_id"));
|
||||
|
||||
return devices;
|
||||
}
|
||||
|
||||
void DataImporter::ImportData(const std::string& table, int device)
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << "Table: " << table << " #" << device;
|
||||
Logging::Log(Logging::Severity::Info, ss.str());
|
||||
|
||||
int maxMySQLPacketSize = GetMaximumMySQLPacketSize() - 2;
|
||||
int maxTimestamp = GetMaximumTimestamp(table, device);
|
||||
int dataId = DataStorage::DataId(table).Id();
|
||||
|
||||
std::stringstream queryPart;
|
||||
|
||||
std::stringstream query;
|
||||
query << "SELECT timestamp, value FROM " << table << " WHERE device_id = '" << device << "' AND timestamp > '" << maxTimestamp << "' ORDER BY timestamp ASC;";
|
||||
auto resultSet = m_pSQLiteClient->ExecuteQuery(query.str());
|
||||
|
||||
int writeCount = 0;
|
||||
InitializeInsertQuery(query);
|
||||
while (resultSet.Next())
|
||||
{
|
||||
queryPart.str("");
|
||||
queryPart << "('" << device << "', '" << dataId << "', FROM_UNIXTIME(" << resultSet.Int("timestamp") << "), '" << resultSet.String("value") << "'),";
|
||||
++writeCount;
|
||||
|
||||
int packetSize = StringStreamSize(query) + StringStreamSize(queryPart);
|
||||
if (packetSize > maxMySQLPacketSize)
|
||||
{
|
||||
ExecuteInsertQuery(query);
|
||||
writeCount = 0;
|
||||
InitializeInsertQuery(query);
|
||||
}
|
||||
|
||||
query << queryPart.str();
|
||||
}
|
||||
|
||||
if (writeCount > 0)
|
||||
ExecuteInsertQuery(query);
|
||||
}
|
||||
|
||||
int DataImporter::GetMaximumMySQLPacketSize()
|
||||
{
|
||||
std::stringstream query;
|
||||
query << "show variables like 'max_allowed_packet';";
|
||||
MySQL::MySQLResultSet resultSet = m_pMySQLClient->ExecuteQuery(query.str());
|
||||
|
||||
if (resultSet.RowsCount() != 1)
|
||||
throw std::runtime_error("Unexpected reply when retrieving max_allowed_packet.");
|
||||
|
||||
resultSet.Next();
|
||||
return resultSet.Int("Value");
|
||||
}
|
||||
|
||||
int DataImporter::StringStreamSize(std::stringstream& ss)
|
||||
{
|
||||
ss.seekp(0, std::ios_base::end);
|
||||
return ss.tellp();
|
||||
}
|
||||
|
||||
void DataImporter::InitializeInsertQuery(std::stringstream& query)
|
||||
{
|
||||
query.str("");
|
||||
query << "INSERT INTO `datalog` (`device_id`, `data_id`, `timestamp`, `value`) VALUES ";
|
||||
}
|
||||
|
||||
void DataImporter::ExecuteInsertQuery(std::stringstream& query)
|
||||
{
|
||||
query.seekp(-1, std::ios_base::end);
|
||||
query << ";";
|
||||
m_pMySQLClient->Execute(query.str());
|
||||
}
|
||||
|
||||
int DataImporter::GetMaximumTimestamp(const std::string& table, int device)
|
||||
{
|
||||
std::stringstream query;
|
||||
query << "SELECT MAX(UNIX_TIMESTAMP(`timestamp`)) AS timestamp FROM `datalog` WHERE `device_id` = '" << device <<"' AND `data_id` = '" << DataStorage::DataId(table).Id() << "';";
|
||||
auto resultSet = m_pMySQLClient->ExecuteQuery(query.str());
|
||||
if (resultSet.Next())
|
||||
return resultSet.Int("timestamp");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace DataInterface
|
||||
} // namespace DataStorageInterface
|
||||
@@ -0,0 +1,49 @@
|
||||
#ifndef DATAINTERFACE_DATAIMPORTER_H
|
||||
#define DATAINTERFACE_DATAIMPORTER_H
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace MySQL {
|
||||
|
||||
class MySQLClient;
|
||||
|
||||
} // namespace MySQL
|
||||
|
||||
namespace SQLite {
|
||||
|
||||
class SQLiteClient;
|
||||
|
||||
} // namespace SQLite
|
||||
|
||||
namespace DataStorageInterface {
|
||||
namespace DataInterface {
|
||||
|
||||
class DataImporter
|
||||
{
|
||||
public:
|
||||
DataImporter(MySQL::MySQLClient* pMySQLClient, SQLite::SQLiteClient* pSQLiteClient);
|
||||
|
||||
void ImportData();
|
||||
|
||||
private:
|
||||
std::vector<std::string> GetTables();
|
||||
std::vector<int> GetDevices(const std::string& table);
|
||||
void ImportData(const std::string& table, int device);
|
||||
|
||||
int GetMaximumMySQLPacketSize();
|
||||
int StringStreamSize(std::stringstream& ss);
|
||||
void InitializeInsertQuery(std::stringstream& query);
|
||||
void ExecuteInsertQuery(std::stringstream& query);
|
||||
int GetMaximumTimestamp(const std::string& table, int device);
|
||||
|
||||
private:
|
||||
MySQL::MySQLClient* m_pMySQLClient;
|
||||
SQLite::SQLiteClient* m_pSQLiteClient;
|
||||
};
|
||||
|
||||
} // namespace DataInterface
|
||||
} // namespace DataStorageInterface
|
||||
|
||||
#endif // DATAINTERFACE_DATAIMPORTER_H
|
||||
@@ -0,0 +1,45 @@
|
||||
#include "DataType.h"
|
||||
#include <StringAlgorithm.h>
|
||||
|
||||
|
||||
namespace DataStorageInterface {
|
||||
namespace DataInterface {
|
||||
|
||||
DataType::type GetDataType(const std::string& value)
|
||||
{
|
||||
if (StringAlgorithm::is_numeric(value))
|
||||
return DataType::Float;
|
||||
|
||||
return DataType::String;
|
||||
}
|
||||
|
||||
namespace Conversions {
|
||||
|
||||
DataType::type DataType(const std::string& datatype)
|
||||
{
|
||||
if (StringAlgorithm::iequals(datatype, "Float"))
|
||||
return DataType::Float;
|
||||
else if (StringAlgorithm::iequals(datatype, "String"))
|
||||
return DataType::String;
|
||||
else
|
||||
return DataType::None;
|
||||
}
|
||||
|
||||
std::string DataType(DataType::type datatype)
|
||||
{
|
||||
switch (datatype)
|
||||
{
|
||||
case DataType::None:
|
||||
return "None";
|
||||
case DataType::Float:
|
||||
return "Float";
|
||||
default:
|
||||
case DataType::String:
|
||||
return "String";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Conversions
|
||||
|
||||
} // namespace DataInterface
|
||||
} // namespace DataStorageInterface
|
||||
@@ -0,0 +1,33 @@
|
||||
#ifndef DATAINTERFACE_DATATYPE_H
|
||||
#define DATAINTERFACE_DATATYPE_H
|
||||
|
||||
#include <string>
|
||||
|
||||
|
||||
namespace DataStorageInterface {
|
||||
namespace DataInterface {
|
||||
|
||||
class DataType
|
||||
{
|
||||
public:
|
||||
enum type
|
||||
{
|
||||
None,
|
||||
Float,
|
||||
String
|
||||
};
|
||||
};
|
||||
|
||||
DataType::type GetDataType(const std::string& value);
|
||||
|
||||
namespace Conversions {
|
||||
|
||||
DataType::type DataType(const std::string& datatype);
|
||||
std::string DataType(DataType::type datatype);
|
||||
|
||||
} // namespace Conversions
|
||||
|
||||
} // namespace DataInterface
|
||||
} // namespace DataStorageInterface
|
||||
|
||||
#endif // DATAINTERFACE_DATATYPE_H
|
||||
@@ -0,0 +1,117 @@
|
||||
#include "GraphClient.h"
|
||||
#include "Util/Util.h"
|
||||
#include <MySQLClient.h>
|
||||
#include <StringAlgorithm.h>
|
||||
#include <sstream>
|
||||
|
||||
|
||||
namespace DataStorageInterface {
|
||||
namespace DataInterface {
|
||||
|
||||
GraphClient::GraphClient(MySQL::MySQLClient* pMySQLClient) :
|
||||
m_pMySQLClient(pMySQLClient)
|
||||
{
|
||||
}
|
||||
|
||||
nlohmann::json GraphClient::GetGraphHeader(const std::vector<int>& dataIds, DataStorage::Timespan::type timespan)
|
||||
{
|
||||
int currentTimestamp = Util::GetTimestamp() + Util::GetUTCOffset();
|
||||
return GetGraphHeader(timespan, currentTimestamp);
|
||||
}
|
||||
|
||||
nlohmann::json GraphClient::GetGraphData(int deviceId, const std::vector<int>& dataIds, DataStorage::Timespan::type timespan)
|
||||
{
|
||||
int currentTimestamp = Util::GetTimestamp() + Util::GetUTCOffset();
|
||||
nlohmann::json json = GetGraphHeader(timespan, currentTimestamp);
|
||||
|
||||
std::string table;
|
||||
switch (timespan)
|
||||
{
|
||||
case DataStorage::Timespan::Day:
|
||||
table = "datalog"; // "datalog-daily"
|
||||
break;
|
||||
case DataStorage::Timespan::Week:
|
||||
table = "datalog-weekly";
|
||||
break;
|
||||
case DataStorage::Timespan::Month:
|
||||
table = "datalog-monthly";
|
||||
break;
|
||||
case DataStorage::Timespan::Year:
|
||||
table = "datalog-yearly";
|
||||
break;
|
||||
default:
|
||||
case DataStorage::Timespan::Unknown:
|
||||
return nlohmann::json();
|
||||
}
|
||||
|
||||
nlohmann::json rootData = nlohmann::json::array();
|
||||
std::stringstream query;
|
||||
|
||||
int startTimestamp = Util::GetTimestamp() - timespan;
|
||||
for (auto dataId : dataIds)
|
||||
{
|
||||
nlohmann::json data;
|
||||
nlohmann::json dataArray;
|
||||
query.str("");
|
||||
|
||||
if (StringAlgorithm::iequals(table, "datalog"))
|
||||
query << "SELECT UNIX_TIMESTAMP(`d`.`timestamp`) AS timestamp, `d`.`value` AS 'min-value', `d`.`value` AS 'mean-value', `d`.`value` AS 'max-value' ";
|
||||
else
|
||||
query << "SELECT UNIX_TIMESTAMP(`d`.`timestamp`) AS timestamp, `d`.`min-value`, `d`.`mean-value`, `d`.`max-value` ";
|
||||
|
||||
query << "FROM `" << table << "` AS `d` ";
|
||||
query << "WHERE `d`.`device_id` = '" << deviceId << "' ";
|
||||
query << "AND `d`.`data_id` = '" << dataId << "' ";
|
||||
query << "AND UNIX_TIMESTAMP(`d`.`timestamp`) > '" << startTimestamp << "';";
|
||||
|
||||
auto resultSet = m_pMySQLClient->ExecuteQuery(query.str());
|
||||
|
||||
int offset = Util::GetUTCOffset();
|
||||
int timestamp;
|
||||
double minValue, meanValue, maxValue;
|
||||
if (resultSet.RowsCount() > 0)
|
||||
{
|
||||
while (resultSet.Next())
|
||||
{
|
||||
timestamp = resultSet.Int("timestamp") + offset;
|
||||
minValue = resultSet.Double("min-value");
|
||||
meanValue = resultSet.Double("mean-value");
|
||||
maxValue = resultSet.Double("max-value");
|
||||
|
||||
nlohmann::json entry = nlohmann::json::array();
|
||||
entry.push_back(timestamp);
|
||||
|
||||
std::stringstream entryValue;
|
||||
if (StringAlgorithm::iequals(table, "datalog"))
|
||||
entryValue << std::fixed << meanValue;
|
||||
else
|
||||
entryValue << std::fixed << minValue << ", " << std::fixed << maxValue;
|
||||
entry.push_back(entryValue.str());
|
||||
|
||||
dataArray.push_back(entry);
|
||||
}
|
||||
|
||||
data["data"] = dataArray;
|
||||
rootData.push_back(data);
|
||||
}
|
||||
}
|
||||
|
||||
json["data"] = rootData;
|
||||
|
||||
return json;
|
||||
}
|
||||
|
||||
nlohmann::json GraphClient::GetGraphHeader(DataStorage::Timespan::type timespan, std::time_t currentTimestamp)
|
||||
{
|
||||
int startTimestamp = currentTimestamp - timespan;
|
||||
|
||||
nlohmann::json json;
|
||||
json["timespan"] = DataStorage::Conversions::Timespan(timespan);
|
||||
json["start"] = startTimestamp;
|
||||
json["end"] = currentTimestamp;
|
||||
|
||||
return json;
|
||||
}
|
||||
|
||||
} // namespace DataInterface
|
||||
} // namespace DataStorageInterface
|
||||
@@ -0,0 +1,37 @@
|
||||
#ifndef DATAINTERFACE_GRAPHCLIENT_H
|
||||
#define DATAINTERFACE_GRAPHCLIENT_H
|
||||
|
||||
#include <Timespan.h>
|
||||
#include <json.hpp>
|
||||
#include <ctime>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace MySQL {
|
||||
|
||||
class MySQLClient;
|
||||
|
||||
} // namespace MySQL
|
||||
|
||||
namespace DataStorageInterface {
|
||||
namespace DataInterface {
|
||||
|
||||
class GraphClient
|
||||
{
|
||||
public:
|
||||
GraphClient(MySQL::MySQLClient* pMySQLClient);
|
||||
|
||||
nlohmann::json GetGraphHeader(const std::vector<int>& dataIds, DataStorage::Timespan::type timespan);
|
||||
nlohmann::json GetGraphData(int deviceId, const std::vector<int>& dataIds, DataStorage::Timespan::type timespan);
|
||||
|
||||
private:
|
||||
nlohmann::json GetGraphHeader(DataStorage::Timespan::type timespan, std::time_t currentTimestamp);
|
||||
|
||||
private:
|
||||
MySQL::MySQLClient* m_pMySQLClient;
|
||||
};
|
||||
|
||||
} // namespace DataInterface
|
||||
} // namespace DataStorageInterface
|
||||
|
||||
#endif // DATAINTERFACE_GRAPHCLIENT_H
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../Makefile
|
||||
@@ -0,0 +1,24 @@
|
||||
#ifndef DATAINTERFACE_RRA_H
|
||||
#define DATAINTERFACE_RRA_H
|
||||
|
||||
|
||||
namespace DataStorageInterface {
|
||||
namespace DataInterface {
|
||||
|
||||
class RRA
|
||||
{
|
||||
public:
|
||||
enum type
|
||||
{
|
||||
Raw = 1,
|
||||
Daily = 300,
|
||||
Weekly = 1800,
|
||||
Monthly = 7200,
|
||||
Yearly = 86400
|
||||
};
|
||||
};
|
||||
|
||||
} // namespace DataInterface
|
||||
} // namespace DataStorageInterface
|
||||
|
||||
#endif // DATAINTERFACE_RRA_H
|
||||
Reference in New Issue
Block a user