Compare commits

..

15 Commits

Author SHA1 Message Date
9473c5a621 Remove unnecessary rewind after file open 2026-02-25 21:08:54 +02:00
953b0bdb9a Remove unused _detect_header_and_buffer method and related fields 2026-02-25 20:56:43 +02:00
f58596ebf7 Refactor FileDatasource: remove unused header detection variables 2026-02-25 12:35:58 +02:00
d621390f51 Refactor file rewind logic to skip header row and remove unnecessary buffers 2026-02-25 12:11:05 +02:00
e4be6b0a19 Refactor file rewinding logic to skip header row after seek(0) 2026-02-25 12:00:39 +02:00
fe66df9b8c Refactor row reading logic for clarity and efficiency 2026-02-25 11:50:56 +02:00
3e0b4762ef Optimize CSV parsing by adding skipinitialspace=True to csv.reader and removing unnecessary strip() calls 2026-02-24 22:11:37 +02:00
75613fd4fc Restore input validation for accelerometer parsing 2026-02-24 20:47:00 +02:00
a25fbfc3ef Simplify accelerometer parsing and remove int16-specific leftovers 2026-02-24 20:14:56 +02:00
ca790e7306 Remove int16 binding from datasource 2026-02-24 19:55:50 +02:00
1643767094 Remove int16 range check per review 2026-02-24 15:41:41 +02:00
3d94bf3008 Parse accelerometer values as int16 (remove float conversion) 2026-02-24 15:12:49 +02:00
c5d98d53cd Add mosquitto runtime folders to gitignore 2026-02-24 14:25:27 +02:00
07a0e906d8 Fix timestamp field in AggregatedData 2026-02-24 14:21:41 +02:00
c974ac32f6 Реалізовую базовий FileReader та переношу sleep до FileDatasource.read() 2026-02-23 22:01:11 +02:00
19 changed files with 227 additions and 645 deletions

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
agent/docker/mosquitto/data/
agent/docker/mosquitto/log/

3
.idea/.gitignore generated vendored
View File

@@ -1,3 +0,0 @@
# Default ignored files
/shelf/
/workspace.xml

15
.idea/IoT-Systems.iml generated
View File

@@ -1,15 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/store" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/.venv" />
</content>
<orderEntry type="jdk" jdkName="Python 3.12 (IoT-Systems)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
</module>

425
.idea/dbnavigator.xml generated
View File

@@ -1,425 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DBNavigator.Project.DDLFileAttachmentManager">
<mappings />
<preferences />
</component>
<component name="DBNavigator.Project.DatabaseAssistantManager">
<assistants />
</component>
<component name="DBNavigator.Project.DatabaseFileManager">
<open-files />
</component>
<component name="DBNavigator.Project.Settings">
<connections />
<browser-settings>
<general>
<display-mode value="TABBED" />
<navigation-history-size value="100" />
<show-object-details value="false" />
<enable-sticky-paths value="true" />
</general>
<filters>
<object-type-filter>
<object-type name="SCHEMA" enabled="true" />
<object-type name="USER" enabled="true" />
<object-type name="ROLE" enabled="true" />
<object-type name="PRIVILEGE" enabled="true" />
<object-type name="CHARSET" enabled="true" />
<object-type name="TABLE" enabled="true" />
<object-type name="VIEW" enabled="true" />
<object-type name="MATERIALIZED_VIEW" enabled="true" />
<object-type name="NESTED_TABLE" enabled="true" />
<object-type name="COLUMN" enabled="true" />
<object-type name="INDEX" enabled="true" />
<object-type name="CONSTRAINT" enabled="true" />
<object-type name="DATASET_TRIGGER" enabled="true" />
<object-type name="DATABASE_TRIGGER" enabled="true" />
<object-type name="SYNONYM" enabled="true" />
<object-type name="SEQUENCE" enabled="true" />
<object-type name="PROCEDURE" enabled="true" />
<object-type name="FUNCTION" enabled="true" />
<object-type name="PACKAGE" enabled="true" />
<object-type name="TYPE" enabled="true" />
<object-type name="TYPE_ATTRIBUTE" enabled="true" />
<object-type name="ARGUMENT" enabled="true" />
<object-type name="JAVA_CLASS" enabled="true" />
<object-type name="JAVA_INNER_CLASS" enabled="true" />
<object-type name="JAVA_FIELD" enabled="true" />
<object-type name="JAVA_METHOD" enabled="true" />
<object-type name="DIMENSION" enabled="true" />
<object-type name="CLUSTER" enabled="true" />
<object-type name="DBLINK" enabled="true" />
<object-type name="CREDENTIAL" enabled="true" />
<object-type name="AI_PROFILE" enabled="true" />
</object-type-filter>
</filters>
<sorting>
<object-type name="COLUMN" sorting-type="NAME" />
<object-type name="FUNCTION" sorting-type="NAME" />
<object-type name="PROCEDURE" sorting-type="NAME" />
<object-type name="ARGUMENT" sorting-type="POSITION" />
<object-type name="TYPE ATTRIBUTE" sorting-type="POSITION" />
</sorting>
<default-editors>
<object-type name="VIEW" editor-type="SELECTION" />
<object-type name="PACKAGE" editor-type="SELECTION" />
<object-type name="TYPE" editor-type="SELECTION" />
</default-editors>
</browser-settings>
<navigation-settings>
<lookup-filters>
<lookup-objects>
<object-type name="SCHEMA" enabled="true" />
<object-type name="USER" enabled="false" />
<object-type name="ROLE" enabled="false" />
<object-type name="PRIVILEGE" enabled="false" />
<object-type name="CHARSET" enabled="false" />
<object-type name="TABLE" enabled="true" />
<object-type name="VIEW" enabled="true" />
<object-type name="MATERIALIZED VIEW" enabled="true" />
<object-type name="INDEX" enabled="true" />
<object-type name="CONSTRAINT" enabled="true" />
<object-type name="DATASET TRIGGER" enabled="true" />
<object-type name="DATABASE TRIGGER" enabled="true" />
<object-type name="SYNONYM" enabled="false" />
<object-type name="SEQUENCE" enabled="true" />
<object-type name="PROCEDURE" enabled="true" />
<object-type name="FUNCTION" enabled="true" />
<object-type name="PACKAGE" enabled="true" />
<object-type name="TYPE" enabled="true" />
<object-type name="JAVA CLASS" enabled="true" />
<object-type name="INNER CLASS" enabled="true" />
<object-type name="JAVA FIELD" enabled="true" />
<object-type name="JAVA METHOD" enabled="true" />
<object-type name="JAVA PARAMETER" enabled="true" />
<object-type name="DIMENSION" enabled="false" />
<object-type name="CLUSTER" enabled="false" />
<object-type name="DBLINK" enabled="false" />
<object-type name="CREDENTIAL" enabled="false" />
</lookup-objects>
<force-database-load value="false" />
<prompt-connection-selection value="true" />
<prompt-schema-selection value="true" />
</lookup-filters>
</navigation-settings>
<dataset-grid-settings>
<general>
<enable-zooming value="true" />
<enable-column-tooltip value="true" />
</general>
<sorting>
<nulls-first value="true" />
<max-sorting-columns value="4" />
</sorting>
<audit-columns>
<column-names value="" />
<visible value="true" />
<editable value="false" />
</audit-columns>
</dataset-grid-settings>
<dataset-editor-settings>
<text-editor-popup>
<active value="false" />
<active-if-empty value="false" />
<data-length-threshold value="100" />
<popup-delay value="1000" />
</text-editor-popup>
<values-actions-popup>
<show-popup-button value="true" />
<element-count-threshold value="1000" />
<data-length-threshold value="250" />
</values-actions-popup>
<general>
<fetch-block-size value="100" />
<fetch-timeout value="30" />
<trim-whitespaces value="true" />
<convert-empty-strings-to-null value="true" />
<select-content-on-cell-edit value="true" />
<large-value-preview-active value="true" />
</general>
<filters>
<prompt-filter-dialog value="true" />
<default-filter-type value="BASIC" />
</filters>
<qualified-text-editor text-length-threshold="300">
<content-types>
<content-type name="Text" enabled="true" />
<content-type name="Properties" enabled="true" />
<content-type name="XML" enabled="true" />
<content-type name="DTD" enabled="true" />
<content-type name="HTML" enabled="true" />
<content-type name="XHTML" enabled="true" />
<content-type name="SQL" enabled="true" />
<content-type name="PL/SQL" enabled="true" />
<content-type name="JSON" enabled="true" />
<content-type name="JSON5" enabled="true" />
<content-type name="YAML" enabled="true" />
</content-types>
</qualified-text-editor>
<record-navigation>
<navigation-target value="VIEWER" />
</record-navigation>
</dataset-editor-settings>
<code-editor-settings>
<general>
<show-object-navigation-gutter value="false" />
<show-spec-declaration-navigation-gutter value="true" />
<enable-spellchecking value="true" />
<enable-reference-spellchecking value="false" />
</general>
<confirmations>
<save-changes value="false" />
<revert-changes value="true" />
<exit-on-changes value="ASK" />
</confirmations>
</code-editor-settings>
<code-completion-settings>
<filters>
<basic-filter>
<filter-element type="RESERVED_WORD" id="keyword" selected="true" />
<filter-element type="RESERVED_WORD" id="function" selected="true" />
<filter-element type="RESERVED_WORD" id="parameter" selected="true" />
<filter-element type="RESERVED_WORD" id="datatype" selected="true" />
<filter-element type="RESERVED_WORD" id="exception" selected="true" />
<filter-element type="OBJECT" id="schema" selected="true" />
<filter-element type="OBJECT" id="role" selected="true" />
<filter-element type="OBJECT" id="user" selected="true" />
<filter-element type="OBJECT" id="privilege" selected="true" />
<user-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="false" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</user-schema>
<public-schema>
<filter-element type="OBJECT" id="table" selected="false" />
<filter-element type="OBJECT" id="view" selected="false" />
<filter-element type="OBJECT" id="materialized view" selected="false" />
<filter-element type="OBJECT" id="index" selected="false" />
<filter-element type="OBJECT" id="constraint" selected="false" />
<filter-element type="OBJECT" id="trigger" selected="false" />
<filter-element type="OBJECT" id="synonym" selected="false" />
<filter-element type="OBJECT" id="sequence" selected="false" />
<filter-element type="OBJECT" id="procedure" selected="false" />
<filter-element type="OBJECT" id="function" selected="false" />
<filter-element type="OBJECT" id="package" selected="false" />
<filter-element type="OBJECT" id="type" selected="false" />
<filter-element type="OBJECT" id="dimension" selected="false" />
<filter-element type="OBJECT" id="cluster" selected="false" />
<filter-element type="OBJECT" id="dblink" selected="false" />
</public-schema>
<any-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</any-schema>
</basic-filter>
<extended-filter>
<filter-element type="RESERVED_WORD" id="keyword" selected="true" />
<filter-element type="RESERVED_WORD" id="function" selected="true" />
<filter-element type="RESERVED_WORD" id="parameter" selected="true" />
<filter-element type="RESERVED_WORD" id="datatype" selected="true" />
<filter-element type="RESERVED_WORD" id="exception" selected="true" />
<filter-element type="OBJECT" id="schema" selected="true" />
<filter-element type="OBJECT" id="user" selected="true" />
<filter-element type="OBJECT" id="role" selected="true" />
<filter-element type="OBJECT" id="privilege" selected="true" />
<user-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</user-schema>
<public-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</public-schema>
<any-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</any-schema>
</extended-filter>
</filters>
<sorting enabled="true">
<sorting-element type="RESERVED_WORD" id="keyword" />
<sorting-element type="RESERVED_WORD" id="datatype" />
<sorting-element type="OBJECT" id="column" />
<sorting-element type="OBJECT" id="table" />
<sorting-element type="OBJECT" id="view" />
<sorting-element type="OBJECT" id="materialized view" />
<sorting-element type="OBJECT" id="index" />
<sorting-element type="OBJECT" id="constraint" />
<sorting-element type="OBJECT" id="trigger" />
<sorting-element type="OBJECT" id="synonym" />
<sorting-element type="OBJECT" id="sequence" />
<sorting-element type="OBJECT" id="procedure" />
<sorting-element type="OBJECT" id="function" />
<sorting-element type="OBJECT" id="package" />
<sorting-element type="OBJECT" id="type" />
<sorting-element type="OBJECT" id="dimension" />
<sorting-element type="OBJECT" id="cluster" />
<sorting-element type="OBJECT" id="dblink" />
<sorting-element type="OBJECT" id="schema" />
<sorting-element type="OBJECT" id="role" />
<sorting-element type="OBJECT" id="user" />
<sorting-element type="RESERVED_WORD" id="function" />
<sorting-element type="RESERVED_WORD" id="parameter" />
</sorting>
<format>
<enforce-code-style-case value="true" />
</format>
</code-completion-settings>
<execution-engine-settings>
<statement-execution>
<fetch-block-size value="100" />
<execution-timeout value="20" />
<debug-execution-timeout value="600" />
<focus-result value="false" />
<prompt-execution value="false" />
</statement-execution>
<script-execution>
<command-line-interfaces />
<execution-timeout value="300" />
</script-execution>
<method-execution>
<execution-timeout value="30" />
<debug-execution-timeout value="600" />
<parameter-history-size value="10" />
</method-execution>
</execution-engine-settings>
<operation-settings>
<transactions>
<uncommitted-changes>
<on-project-close value="ASK" />
<on-disconnect value="ASK" />
<on-autocommit-toggle value="ASK" />
</uncommitted-changes>
<multiple-uncommitted-changes>
<on-commit value="ASK" />
<on-rollback value="ASK" />
</multiple-uncommitted-changes>
</transactions>
<session-browser>
<disconnect-session value="ASK" />
<kill-session value="ASK" />
<reload-on-filter-change value="false" />
</session-browser>
<compiler>
<compile-type value="KEEP" />
<compile-dependencies value="ASK" />
<always-show-controls value="false" />
</compiler>
</operation-settings>
<ddl-file-settings>
<extensions>
<mapping file-type-id="VIEW" extensions="vw" />
<mapping file-type-id="TRIGGER" extensions="trg" />
<mapping file-type-id="PROCEDURE" extensions="prc" />
<mapping file-type-id="FUNCTION" extensions="fnc" />
<mapping file-type-id="PACKAGE" extensions="pkg" />
<mapping file-type-id="PACKAGE_SPEC" extensions="pks" />
<mapping file-type-id="PACKAGE_BODY" extensions="pkb" />
<mapping file-type-id="TYPE" extensions="tpe" />
<mapping file-type-id="TYPE_SPEC" extensions="tps" />
<mapping file-type-id="TYPE_BODY" extensions="tpb" />
<mapping file-type-id="JAVA_SOURCE" extensions="sql" />
</extensions>
<general>
<lookup-ddl-files value="true" />
<create-ddl-files value="false" />
<synchronize-ddl-files value="true" />
<use-qualified-names value="false" />
<make-scripts-rerunnable value="true" />
</general>
</ddl-file-settings>
<assistant-settings>
<credential-settings>
<credentials />
</credential-settings>
</assistant-settings>
<general-settings>
<regional-settings>
<date-format value="MEDIUM" />
<number-format value="UNGROUPED" />
<locale value="SYSTEM_DEFAULT" />
<use-custom-formats value="false" />
</regional-settings>
<environment>
<environment-types>
<environment-type id="development" name="Development" description="Development environment" color="-2430209/-12296320" readonly-code="false" readonly-data="false" />
<environment-type id="integration" name="Integration" description="Integration environment" color="-2621494/-12163514" readonly-code="true" readonly-data="false" />
<environment-type id="production" name="Production" description="Productive environment" color="-11574/-10271420" readonly-code="true" readonly-data="true" />
<environment-type id="other" name="Other" description="" color="-1576/-10724543" readonly-code="false" readonly-data="false" />
</environment-types>
<visibility-settings>
<connection-tabs value="true" />
<dialog-headers value="true" />
<object-editor-tabs value="true" />
<script-editor-tabs value="false" />
<execution-result-tabs value="true" />
</visibility-settings>
</environment>
</general-settings>
</component>
</project>

View File

@@ -1,16 +0,0 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyInterpreterInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="2">
<item index="0" class="java.lang.String" itemvalue="requests" />
<item index="1" class="java.lang.String" itemvalue="python-telegram-bot" />
</list>
</value>
</option>
</inspection_tool>
</profile>
</component>

View File

@@ -1,6 +0,0 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

7
.idea/misc.xml generated
View File

@@ -1,7 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="Python 3.12 (IoT-Systems)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (IoT-Systems)" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml generated
View File

@@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/IoT-Systems.iml" filepath="$PROJECT_DIR$/.idea/IoT-Systems.iml" />
</modules>
</component>
</project>

6
.idea/vcs.xml generated
View File

@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

View File

@@ -1,3 +1,4 @@
version: "3.9"
name: "road_vision"
services:
mqtt:

View File

@@ -1,5 +1,9 @@
from csv import reader
import csv
import time
from datetime import datetime
from pathlib import Path
from typing import Optional, List
from domain.accelerometer import Accelerometer
from domain.gps import Gps
from domain.aggregated_data import AggregatedData
@@ -7,24 +11,141 @@ import config
class FileDatasource:
def __init__(
self,
accelerometer_filename: str,
gps_filename: str,
) -> None:
pass
def read(self) -> AggregatedData:
"""Метод повертає дані отримані з датчиків"""
return AggregatedData(
Accelerometer(1, 2, 3),
Gps(4, 5),
datetime.now(),
config.USER_ID,
)
def __init__(self, accelerometer_filename: str, gps_filename: str) -> None:
self.accelerometer_filename = accelerometer_filename
self.gps_filename = gps_filename
self._acc_f = None
self._gps_f = None
self._acc_reader: Optional[csv.reader] = None
self._gps_reader: Optional[csv.reader] = None
self._started = False
def startReading(self, *args, **kwargs):
"""Метод повинен викликатись перед початком читання даних"""
"""Must be called before read()"""
if self._started:
return
if not Path(self.accelerometer_filename).exists():
raise FileNotFoundError(f"Accelerometer file not found: {self.accelerometer_filename}")
if not Path(self.gps_filename).exists():
raise FileNotFoundError(f"GPS file not found: {self.gps_filename}")
self._open_files()
self._started = True
def stopReading(self, *args, **kwargs):
"""Метод повинен викликатись для закінчення читання даних"""
"""Must be called when finishing reading"""
self._close_files()
self._started = False
def read(self) -> AggregatedData:
"""Return one combined sample (acc + gps)."""
if not self._started:
raise RuntimeError("Datasource is not started. Call startReading() before read().")
acc_row = self._get_next_row(self._acc_reader, source="acc")
gps_row = self._get_next_row(self._gps_reader, source="gps")
acc = self._parse_acc(acc_row)
gps = self._parse_gps(gps_row)
# IMPORTANT: timing belongs to datasource (not MQTT / main.py)
if config.DELAY and config.DELAY > 0:
time.sleep(float(config.DELAY))
return AggregatedData(
accelerometer=acc,
gps=gps,
timestamp=datetime.utcnow(),
user_id=config.USER_ID,
)
# ---------------- internal ----------------
def _open_files(self) -> None:
self._close_files()
self._acc_f = open(self.accelerometer_filename, "r", newline="", encoding="utf-8")
self._gps_f = open(self.gps_filename, "r", newline="", encoding="utf-8")
self._acc_reader = csv.reader(self._acc_f, skipinitialspace=True)
self._gps_reader = csv.reader(self._gps_f, skipinitialspace=True)
# File pointer is already at 0 right after open(), so no need to rewind here.
# Skip header row once.
next(self._acc_reader, None)
next(self._gps_reader, None)
def _close_files(self) -> None:
for f in (self._acc_f, self._gps_f):
try:
if f is not None:
f.close()
except Exception:
pass
self._acc_f = None
self._gps_f = None
self._acc_reader = None
self._gps_reader = None
def _rewind_acc(self) -> None:
if self._acc_f is None:
raise RuntimeError("Accelerometer file is not open.")
self._acc_f.seek(0)
self._acc_reader = csv.reader(self._acc_f, skipinitialspace=True)
next(self._acc_reader, None) # skip header row
def _rewind_gps(self) -> None:
if self._gps_f is None:
raise RuntimeError("GPS file is not open.")
self._gps_f.seek(0)
self._gps_reader = csv.reader(self._gps_f, skipinitialspace=True)
next(self._gps_reader, None) # skip header row
def _get_next_row(self, reader, source: str) -> List[str]:
"""Get the next valid row from the reader."""
if reader is None:
raise RuntimeError("Reader is not initialized.")
while True:
row = next(reader, None)
if row is None:
# EOF -> rewind & continue
if source == "acc":
self._rewind_acc()
reader = self._acc_reader
else:
self._rewind_gps()
reader = self._gps_reader
continue
if not row or not any(cell.strip() for cell in row):
continue
return row
@staticmethod
def _parse_acc(row: List[str]) -> Accelerometer:
if len(row) < 3:
raise ValueError(f"Accelerometer row must have 3 values (x,y,z). Got: {row}")
try:
x = int(row[0])
y = int(row[1])
z = int(row[2])
except ValueError as e:
raise ValueError(f"Invalid accelerometer values (expected integers): {row}") from e
return Accelerometer(x=x, y=y, z=z)
@staticmethod
def _parse_gps(row: List[str]) -> Gps:
if len(row) < 2:
raise ValueError(f"GPS row must have 2 values (longitude,latitude). Got: {row}")
lon = float(row[0])
lat = float(row[1])
return Gps(longitude=lon, latitude=lat)

View File

@@ -1,6 +1,4 @@
from paho.mqtt import client as mqtt_client
import json
import time
from schema.aggregated_data_schema import AggregatedDataSchema
from file_datasource import FileDatasource
import config
@@ -24,19 +22,14 @@ def connect_mqtt(broker, port):
return client
def publish(client, topic, datasource, delay):
def publish(client, topic, datasource):
datasource.startReading()
while True:
time.sleep(delay)
data = datasource.read()
msg = AggregatedDataSchema().dumps(data)
result = client.publish(topic, msg)
# result: [0, 1]
status = result[0]
if status == 0:
pass
# print(f"Send `{msg}` to topic `{topic}`")
else:
if status != 0:
print(f"Failed to send message to topic {topic}")
@@ -44,9 +37,9 @@ def run():
# Prepare mqtt client
client = connect_mqtt(config.MQTT_BROKER_HOST, config.MQTT_BROKER_PORT)
# Prepare datasource
datasource = FileDatasource("data/data.csv", "data/gps_data.csv")
datasource = FileDatasource("data/accelerometer.csv", "data/gps.csv")
# Infinity publish data
publish(client, config.MQTT_TOPIC, datasource, config.DELAY)
publish(client, config.MQTT_TOPIC, datasource)
if __name__ == "__main__":

2
store/.gitignore vendored
View File

@@ -1,5 +1,3 @@
venv
__pycache__
.idea
.idea/

View File

View File

@@ -1,15 +0,0 @@
from sqlalchemy import MetaData
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, declarative_base
from config import POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB
DATABASE_URL = f"postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}"
engine = create_engine(DATABASE_URL)
Base = declarative_base()
metadata = MetaData()
SessionLocal = sessionmaker(bind=engine)

View File

@@ -1,7 +1,8 @@
version: "3.9"
name: "road_vision__database"
services:
postgres_db:
image: postgres:17
image: postgres:latest
container_name: postgres_db
restart: always
environment:

View File

@@ -1,8 +1,10 @@
import asyncio
import json
from typing import Set, Dict, List
from typing import Set, Dict, List, Any
from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect, Body
from fastapi.encoders import jsonable_encoder
from sqlalchemy import (
create_engine,
MetaData,
Table,
Column,
Integer,
@@ -10,14 +12,25 @@ from sqlalchemy import (
Float,
DateTime,
)
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql import select
from database import metadata, SessionLocal
from schemas import ProcessedAgentData, ProcessedAgentDataInDB
from datetime import datetime
from pydantic import BaseModel, field_validator
from config import (
POSTGRES_HOST,
POSTGRES_PORT,
POSTGRES_DB,
POSTGRES_USER,
POSTGRES_PASSWORD,
)
# FastAPI app setup
app = FastAPI()
# SQLAlchemy setup
DATABASE_URL = f"postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}"
engine = create_engine(DATABASE_URL)
metadata = MetaData()
# Define the ProcessedAgentData table
processed_agent_data = Table(
"processed_agent_data",
metadata,
@@ -31,6 +44,57 @@ processed_agent_data = Table(
Column("longitude", Float),
Column("timestamp", DateTime),
)
SessionLocal = sessionmaker(bind=engine)
# SQLAlchemy model
class ProcessedAgentDataInDB(BaseModel):
id: int
road_state: str
user_id: int
x: float
y: float
z: float
latitude: float
longitude: float
timestamp: datetime
# FastAPI models
class AccelerometerData(BaseModel):
x: float
y: float
z: float
class GpsData(BaseModel):
latitude: float
longitude: float
class AgentData(BaseModel):
user_id: int
accelerometer: AccelerometerData
gps: GpsData
timestamp: datetime
@classmethod
@field_validator("timestamp", mode="before")
def check_timestamp(cls, value):
if isinstance(value, datetime):
return value
try:
return datetime.fromisoformat(value)
except (TypeError, ValueError):
raise ValueError(
"Invalid timestamp format. Expected ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)."
)
class ProcessedAgentData(BaseModel):
road_state: str
agent_data: AgentData
# WebSocket subscriptions
subscriptions: Dict[int, Set[WebSocket]] = {}
@@ -61,36 +125,10 @@ async def send_data_to_subscribers(user_id: int, data):
@app.post("/processed_agent_data/")
async def create_processed_agent_data(data: List[ProcessedAgentData], user_id: int = Body(..., embed=True)):
session = SessionLocal()
try:
created_data = [
{
"road_state": item.road_state,
"user_id": user_id,
"x": item.agent_data.accelerometer.x,
"y": item.agent_data.accelerometer.y,
"z": item.agent_data.accelerometer.z,
"latitude": item.agent_data.gps.latitude,
"longitude": item.agent_data.gps.longitude,
"timestamp": item.agent_data.timestamp,
}
for item in data
]
stmt = processed_agent_data.insert().values(created_data).returning(processed_agent_data)
result = session.execute(stmt)
created_records = [dict(row._mapping) for row in result.fetchall()]
session.commit()
for record in created_records:
await send_data_to_subscribers(user_id, jsonable_encoder(record))
return created_records
except Exception as err:
session.rollback()
print(f"Database error: {err}")
raise HTTPException(status_code=500, detail="Internal Server Error")
finally:
session.close()
async def create_processed_agent_data(data: List[ProcessedAgentData]):
# Insert data to database
# Send data to subscribers
pass
@app.get(
@@ -98,34 +136,14 @@ async def create_processed_agent_data(data: List[ProcessedAgentData], user_id: i
response_model=ProcessedAgentDataInDB,
)
def read_processed_agent_data(processed_agent_data_id: int):
session = SessionLocal()
try:
stmt = select(processed_agent_data).where(
processed_agent_data.c.id == processed_agent_data_id
)
res = session.execute(stmt).fetchone()
if not res:
raise HTTPException(status_code=404, detail="Not found")
return dict(res._mapping)
finally:
session.close()
# Get data by id
pass
@app.get("/processed_agent_data/", response_model=list[ProcessedAgentDataInDB])
def list_processed_agent_data():
session = SessionLocal()
try:
stmt = select(processed_agent_data)
res = session.execute(stmt).fetchall()
if not res:
raise HTTPException(status_code=404, detail="Not found")
return [dict(r._mapping) for r in res]
finally:
session.close()
# Get list of data
pass
@app.put(

Binary file not shown.

View File

@@ -1,51 +0,0 @@
from datetime import datetime
from pydantic import BaseModel, field_validator
class ProcessedAgentDataInDB(BaseModel):
id: int
road_state: str
user_id: int
x: float
y: float
z: float
latitude: float
longitude: float
timestamp: datetime
# FastAPI models
class AccelerometerData(BaseModel):
x: float
y: float
z: float
class GpsData(BaseModel):
latitude: float
longitude: float
class AgentData(BaseModel):
user_id: int
accelerometer: AccelerometerData
gps: GpsData
timestamp: datetime
@classmethod
@field_validator("timestamp", mode="before")
def check_timestamp(cls, value):
if isinstance(value, datetime):
return value
try:
return datetime.fromisoformat(value)
except (TypeError, ValueError):
raise ValueError(
"Invalid timestamp format. Expected ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)."
)
class ProcessedAgentData(BaseModel):
road_state: str
agent_data: AgentData