JDierkse/DataStorageInterface
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Makefile.conf and fix Line Endings

Browse Source
This commit is contained in:
JDierkse
2020-05-04 21:45:25 +02:00
parent 9b66a7dd1f
commit 94739511dd
15 changed files with 952 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DataCondenser.ini
-1
View File
@@ -5,4 +5,3 @@ Hostname =
Username = Username =
Password = Password =
Database = Database =
DataImporter.ini
-1
View File
@@ -5,4 +5,3 @@ Hostname =
Username = Username =
Password = Password =
Database = Database =
DataInterface/DataCondenser.cpp
+2 -40
View File
@@ -1,7 +1,6 @@
#include "DataCondenser.h" #include "DataCondenser.h"
#include "Util/Util.h"
#include <DataId.h> #include <DataId.h>
#include <iostream> #include <DateTime.h>
#include <limits> #include <limits>
#include <map> #include <map>
#include <memory> #include <memory>
@@ -22,18 +21,12 @@ void DataCondenser::CondenseData()
for (int dataId = DataStorage::DataId::Min(); dataId <= DataStorage::DataId::Max(); ++dataId) for (int dataId = DataStorage::DataId::Min(); dataId <= DataStorage::DataId::Max(); ++dataId)
{ {
std::cout << "DataID: " << dataId << std::endl;
//std::cout << "Daily" << std::endl;
//for (auto& deviceId : deviceIDs) //for (auto& deviceId : deviceIDs)
// WriteReducedData(deviceId, dataId, RRA::Daily); // WriteReducedData(deviceId, dataId, RRA::Daily);
std::cout << "Weekly" << std::endl;
for (auto& deviceId : deviceIDs) for (auto& deviceId : deviceIDs)
WriteReducedData(deviceId, dataId, RRA::Weekly); WriteReducedData(deviceId, dataId, RRA::Weekly);
std::cout << "Monthly" << std::endl;
for (auto& deviceId : deviceIDs) for (auto& deviceId : deviceIDs)
WriteReducedData(deviceId, dataId, RRA::Monthly); WriteReducedData(deviceId, dataId, RRA::Monthly);
std::cout << "Yearly" << std::endl;
for (auto& deviceId : deviceIDs) for (auto& deviceId : deviceIDs)
WriteReducedData(deviceId, dataId, RRA::Yearly); WriteReducedData(deviceId, dataId, RRA::Yearly);
} }
@@ -176,13 +169,6 @@ void DataCondenser::WriteReducedData(int deviceId, int dataId, const RRA::type&
{ {
DataType::type dataType = GetDataType(deviceId, dataId); DataType::type dataType = GetDataType(deviceId, dataId);
if (dataType == DataType::None)
std::cout << " DataType: None" << std::endl;
else if (dataType == DataType::Float)
std::cout << " DataType: Float" << std::endl;
else if (dataType == DataType::String)
std::cout << " DataType: String" << std::endl;
switch (dataType) switch (dataType)
{ {
case DataType::Float: case DataType::Float:
@@ -207,11 +193,6 @@ void DataCondenser::WriteReducedFloatData(int deviceId, int dataId, const RRA::t
int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId); int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId);
int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId); int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId);
std::cout << " DeviceID: " << deviceId << std::endl;
std::cout << " DataID: " << dataId << std::endl;
std::cout << " Newest Src : " << newestSourcePeriodStamp << std::endl;
std::cout << " Newest Dest: " << newestDestinationPeriodStamp << std::endl;
std::stringstream query; std::stringstream query;
MySQL::MySQLResultSet resultSet; MySQL::MySQLResultSet resultSet;
if (sourceTable == "datalog") if (sourceTable == "datalog")
@@ -219,8 +200,6 @@ void DataCondenser::WriteReducedFloatData(int deviceId, int dataId, const RRA::t
else else
resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp); resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
std::cout << " Source Count: " << resultSet.RowsCount() << std::endl;
int samples = 0; int samples = 0;
float min = std::numeric_limits<float>::max(); float min = std::numeric_limits<float>::max();
float mean = 0; float mean = 0;
@@ -251,7 +230,6 @@ void DataCondenser::WriteReducedFloatData(int deviceId, int dataId, const RRA::t
{ {
ExecuteInsertQuery(query); ExecuteInsertQuery(query);
InitializeInsertQuery(query, destinationTable); InitializeInsertQuery(query, destinationTable);
std::cout << " Write Count: " << writeCount << std::endl;
writeCount = 0; writeCount = 0;
} }
@@ -283,8 +261,6 @@ void DataCondenser::WriteReducedFloatData(int deviceId, int dataId, const RRA::t
if (writeCount > 0) if (writeCount > 0)
ExecuteInsertQuery(query); ExecuteInsertQuery(query);
std::cout << " Write Count: " << writeCount << std::endl << std::endl;
} }
void DataCondenser::WriteReducedStringData(int deviceId, int dataId, const RRA::type& rra) void DataCondenser::WriteReducedStringData(int deviceId, int dataId, const RRA::type& rra)
@@ -297,11 +273,6 @@ void DataCondenser::WriteReducedStringData(int deviceId, int dataId, const RRA::
int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId); int newestSourcePeriodStamp = GetNewestPeriodStamp(sourceTable, deviceId, dataId);
int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId); int newestDestinationPeriodStamp = GetNewestPeriodStamp(destinationTable, deviceId, dataId);
std::cout << " DeviceID: " << deviceId << std::endl;
std::cout << " DataID: " << dataId << std::endl;
std::cout << " Newest Src : " << newestSourcePeriodStamp << std::endl;
std::cout << " Newest Dest: " << newestDestinationPeriodStamp << std::endl;
std::stringstream query; std::stringstream query;
MySQL::MySQLResultSet resultSet; MySQL::MySQLResultSet resultSet;
if (sourceTable == "datalog") if (sourceTable == "datalog")
@@ -309,8 +280,6 @@ void DataCondenser::WriteReducedStringData(int deviceId, int dataId, const RRA::
else else
resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp); resultSet = GetReducedData(sourceTable, deviceId, dataId, newestDestinationPeriodStamp);
std::cout << " Source Count: " << resultSet.RowsCount() << std::endl;
int samples = 0; int samples = 0;
std::vector<std::string> values; std::vector<std::string> values;
@@ -353,7 +322,6 @@ void DataCondenser::WriteReducedStringData(int deviceId, int dataId, const RRA::
{ {
ExecuteInsertQuery(query); ExecuteInsertQuery(query);
InitializeInsertQuery(query, destinationTable); InitializeInsertQuery(query, destinationTable);
std::cout << " Write Count: " << writeCount << std::endl;
writeCount = 0; writeCount = 0;
} }
@@ -376,14 +344,10 @@ void DataCondenser::WriteReducedStringData(int deviceId, int dataId, const RRA::
if (writeCount > 0) if (writeCount > 0)
ExecuteInsertQuery(query); ExecuteInsertQuery(query);
std::cout << " Write Count: " << writeCount << std::endl << std::endl;
} }
void DataCondenser::CleanData() void DataCondenser::CleanData()
{ {
std::cout << "Cleaning: " << std::endl;
CleanData(RRA::Daily); CleanData(RRA::Daily);
CleanData(RRA::Weekly); CleanData(RRA::Weekly);
CleanData(RRA::Monthly); CleanData(RRA::Monthly);
@@ -394,9 +358,7 @@ void DataCondenser::CleanData(const RRA::type& rra)
{ {
std::string table = GetDestinationTable(rra); std::string table = GetDestinationTable(rra);
int periodInSeconds = static_cast<int>(rra) * 400; int periodInSeconds = static_cast<int>(rra) * 400;
int timestamp = Util::GetTimestamp() + Util::GetUTCOffset() - periodInSeconds; int timestamp = DateTime::GetTimestamp() + DateTime::GetUTCOffset() - periodInSeconds;
std::cout << " Cleaning " << table << " (<" << timestamp << ")" << std::endl;
std::stringstream query; std::stringstream query;
query << "DELETE FROM `" << table << "` WHERE `timestamp` < FROM_UNIXTIME(" << timestamp << ");"; query << "DELETE FROM `" << table << "` WHERE `timestamp` < FROM_UNIXTIME(" << timestamp << ");";
DataInterface/GraphClient.cpp
+5 -5
View File
@@ -1,5 +1,5 @@
#include "GraphClient.h" #include "GraphClient.h"
#include "Util/Util.h" #include <DateTime.h>
#include <MySQLClient.h> #include <MySQLClient.h>
#include <StringAlgorithm.h> #include <StringAlgorithm.h>
#include <sstream> #include <sstream>
@@ -15,13 +15,13 @@ GraphClient::GraphClient(MySQL::MySQLClient* pMySQLClient) :
nlohmann::json GraphClient::GetGraphHeader(const std::vector<int>& dataIds, DataStorage::Timespan::type timespan) nlohmann::json GraphClient::GetGraphHeader(const std::vector<int>& dataIds, DataStorage::Timespan::type timespan)
{ {
int currentTimestamp = Util::GetTimestamp() + Util::GetUTCOffset(); int currentTimestamp = DateTime::GetTimestamp() + DateTime::GetUTCOffset();
return GetGraphHeader(timespan, currentTimestamp); return GetGraphHeader(timespan, currentTimestamp);
} }
nlohmann::json GraphClient::GetGraphData(int deviceId, const std::vector<int>& dataIds, DataStorage::Timespan::type timespan) nlohmann::json GraphClient::GetGraphData(int deviceId, const std::vector<int>& dataIds, DataStorage::Timespan::type timespan)
{ {
int currentTimestamp = Util::GetTimestamp() + Util::GetUTCOffset(); int currentTimestamp = DateTime::GetTimestamp() + DateTime::GetUTCOffset();
nlohmann::json json = GetGraphHeader(timespan, currentTimestamp); nlohmann::json json = GetGraphHeader(timespan, currentTimestamp);
std::string table; std::string table;
@@ -47,7 +47,7 @@ nlohmann::json GraphClient::GetGraphData(int deviceId, const std::vector<int>& d
nlohmann::json rootData = nlohmann::json::array(); nlohmann::json rootData = nlohmann::json::array();
std::stringstream query; std::stringstream query;
int startTimestamp = Util::GetTimestamp() - timespan; int startTimestamp = DateTime::GetTimestamp() - timespan;
for (auto dataId : dataIds) for (auto dataId : dataIds)
{ {
nlohmann::json data; nlohmann::json data;
@@ -66,7 +66,7 @@ nlohmann::json GraphClient::GetGraphData(int deviceId, const std::vector<int>& d
auto resultSet = m_pMySQLClient->ExecuteQuery(query.str()); auto resultSet = m_pMySQLClient->ExecuteQuery(query.str());
int offset = Util::GetUTCOffset(); int offset = DateTime::GetUTCOffset();
int timestamp; int timestamp;
double minValue, meanValue, maxValue; double minValue, meanValue, maxValue;
if (resultSet.RowsCount() > 0) if (resultSet.RowsCount() > 0)
Libraries/DateTime
Symlink
+1
View File
@@ -0,0 +1 @@
../../Libraries/DateTime
Makefile.conf
+8 -2
View File
@@ -14,7 +14,7 @@ LFLAGS += -lDataStorage
LFLAGS += -L$(ROOTPATH)/Libraries/DataStorage/lib/$(ARCH) LFLAGS += -L$(ROOTPATH)/Libraries/DataStorage/lib/$(ARCH)
CFLAGS += -I$(ROOTPATH)/Libraries/DataStorage/include CFLAGS += -I$(ROOTPATH)/Libraries/DataStorage/include
LFLAGS += -lHttp LFLAGS += -lHttp -lcrypto -lcurl -lssl -lz
LFLAGS += -L$(ROOTPATH)/Libraries/Http/lib/$(ARCH) LFLAGS += -L$(ROOTPATH)/Libraries/Http/lib/$(ARCH)
CFLAGS += -I$(ROOTPATH)/Libraries/Http/include CFLAGS += -I$(ROOTPATH)/Libraries/Http/include
@@ -26,10 +26,16 @@ LFLAGS += -lUtilities
LFLAGS += -L$(ROOTPATH)/Libraries/Utilities/lib/$(ARCH) LFLAGS += -L$(ROOTPATH)/Libraries/Utilities/lib/$(ARCH)
CFLAGS += -I$(ROOTPATH)/Libraries/Utilities/include CFLAGS += -I$(ROOTPATH)/Libraries/Utilities/include
LFLAGS += -lDateTime
LFLAGS += -L$(ROOTPATH)/Libraries/DateTime/lib/$(ARCH)
CFLAGS += -I$(ROOTPATH)/Libraries/DateTime/include
CFLAGS += -I$(ROOTPATH)/Libraries/clipp/include CFLAGS += -I$(ROOTPATH)/Libraries/clipp/include
CFLAGS += -I$(ROOTPATH)/Libraries/inih CFLAGS += -I$(ROOTPATH)/Libraries/inih
CFLAGS += -I$(ROOTPATH)/Libraries/json/include/nlohmann -I$(ROOTPATH)/Libraries/json/include CFLAGS += -I$(ROOTPATH)/Libraries/json/include/nlohmann -I$(ROOTPATH)/Libraries/json/include
LFLAGS += -pthread -lm LFLAGS += -pthread
CFLAGS += -I$(ROOTPATH) -I$(ROOTPATH)/Libraries CFLAGS += -I$(ROOTPATH) -I$(ROOTPATH)/Libraries
DEBUGDIR := .debug DEBUGDIR := .debug
Util/Makefile
-1
View File
@@ -1 +0,0 @@
../Makefile
Util/Util.cpp
-23
View File
@@ -1,23 +0,0 @@
#include "Util.h"
#include <algorithm>
#include <ctime>
namespace DataStorageInterface {
namespace Util {
int GetTimestamp()
{
return std::time(nullptr);
}
int GetUTCOffset()
{
std::time_t current_time;
std::time(&current_time);
struct std::tm *timeinfo = std::localtime(&current_time);
return timeinfo->tm_gmtoff;
}
} // namespace Util
} // namespace DataStorageInterface
Util/Util.h
-16
View File
@@ -1,16 +0,0 @@
#ifndef UTIL_UTIL_H
#define UTIL_UTIL_H
#include <string>
namespace DataStorageInterface {
namespace Util {
int GetTimestamp();
int GetUTCOffset();
} // namespace Util
} // namespace DataStorageInterface
#endif // UTIL_UTIL_H