[P] Always execute all steps while cleaning up

Add test for new records in DB

.gitea/workflows/reset-docker.yaml

@@ -7,10 +7,13 @@ jobs:
     name: Reset docker state
     steps:
       - name: Stop all containers
+        if: ${{always()}}
         run: docker stop $(docker ps -a | cut -d " " -f 1 | tail -n +2)
       
       - name: Remove all containers
+        if: ${{always()}}
         run: docker rm $(docker ps -a | cut -d " " -f 1 | tail -n +2)
 
       - name: Remove extra volumes
+        if: ${{always()}}
         run: docker volume rm road_vision_postgres_data road_vision_pgadmin-data

.gitea/workflows/tests.yaml

@@ -28,18 +28,27 @@ jobs:
       - name: Clone repository
         run: git clone --revision ${{ gitea.sha }} --depth 1 ${{ gitea.server_url }}/${{ gitea.repository }}
 
+      - name: Start postgres_db container for testing
+        working-directory: IoT-Systems
+        run: docker-compose up -d postgres_db
+
       - name: Build Store testing container
         working-directory: IoT-Systems
         run: docker build -t local/store/${{gitea.sha}} -f store/Dockerfile-test .
 
       - name: Run Store tests
         working-directory: IoT-Systems
-        run: docker run --rm -it local/store/${{gitea.sha}}
+        run: docker run --network host --rm -it local/store/${{gitea.sha}}
 
       - name: Clean up containers
         if: ${{always()}}
         run: docker image rm local/store/${{gitea.sha}}
 
+      - name: Clean up docker-compose
+        if: ${{always()}}
+        working-directory: IoT-Systems
+        run: docker-compose down -v --remove-orphans
+
   integration-smoke-test:
     name: Integration smoke testing
     runs-on: host-arch-x86_64

MapView/datasource.py

@@ -9,6 +9,7 @@ from config import STORE_HOST, STORE_PORT
 
 # Pydantic models
 class ProcessedAgentData(BaseModel):
+    id: int
     road_state: str
     user_id: int
     x: float
@@ -32,9 +33,9 @@ class ProcessedAgentData(BaseModel):
 
 
 class Datasource:
-    def __init__(self, user_id: int):
+    def __init__(self):
+        self.websocket: Connection | None = None
         self.index = 0
-        self.user_id = user_id
         self.connection_status = None
         self._new_points = []
         self._active_markers = []
@@ -47,11 +48,12 @@ class Datasource:
         return points
 
     async def connect_to_server(self):
-        uri = f"ws://{STORE_HOST}:{STORE_PORT}/ws/{self.user_id}"
+        uri = f"ws://{STORE_HOST}:{STORE_PORT}/ws"
         while True:
             Logger.debug("CONNECT TO SERVER")
             async with websockets.connect(uri) as websocket:
                 self.connection_status = "Connected"
+                self.websocket = websocket
                 try:
                     while True:
                         data = await websocket.recv()
@@ -61,8 +63,14 @@ class Datasource:
                     self.connection_status = "Disconnected"
                     Logger.debug("SERVER DISCONNECT")
 
-    def map_lat_lon_to_ProcessedAgentData(self, lat: float, lon: float) -> ProcessedAgentData | None:
-        distances = tuple((abs(lon - marker.latitude) ** 2 + abs(lat - marker.longitude) ** 2) ** 0.5 for marker in self._active_markers)
+    def update_db_record_visibility(self, record_id):
+        if self.websocket:
+            data = json.dumps({"id": record_id})
+            asyncio.ensure_future(self.websocket.send(data))
+
+    def map_lat_lon_to_processed_agent_data(self, lat: float, lon: float) -> ProcessedAgentData | None:
+        distances = tuple((abs(lon - marker.latitude) ** 2 + abs(lat - marker.longitude) ** 2) ** 0.5 for marker in
+                          self._active_markers)
 
         if len(distances) == 0:
             return None

MapView/main.py

@@ -33,7 +33,7 @@ class MapViewApp(App):
         super().__init__(**kwargs)
 
         self.mapview: MapView | None = None
-        self.datasource = Datasource(user_id=1)
+        self.datasource = Datasource()
         self.line_layers = dict()
         self.car_markers = dict()
 
@@ -63,7 +63,7 @@ class MapViewApp(App):
 
             # Оновлює лінію маршрута
             if user_id not in self.line_layers:
-                self.line_layers[user_id] = LineMapLayer(color = line_layer_colors[user_id % len(line_layer_colors)])
+                self.line_layers[user_id] = LineMapLayer(color=line_layer_colors[user_id % len(line_layer_colors)])
                 self.mapview.add_layer(self.line_layers[user_id])
 
             self.line_layers[user_id].add_point((lat, lon))
@@ -106,7 +106,11 @@ class MapViewApp(App):
 
     def map_lat_lon_to_marker(self, lat: float, lon: float) -> MapMarker | None:
         flt = filter(lambda marker: lon == marker.lat and lat == marker.lon, self.pothole_markers + self.bump_markers)
+
+        try:
             return next(flt)
+        except StopIteration:
+            return None
 
     def set_pothole_marker(self, point):
         lat, lon = get_lat_lon(point)
@@ -141,22 +145,24 @@ class MapViewApp(App):
         if lat is None or lon is None:
             return
 
-        clicked_marker_data = self.datasource.map_lat_lon_to_ProcessedAgentData(lat, lon)
+        clicked_marker_data = self.datasource.map_lat_lon_to_processed_agent_data(lat, lon)
 
         if not clicked_marker_data:
             return
 
         clicked_marker = self.map_lat_lon_to_marker(clicked_marker_data.latitude, clicked_marker_data.longitude)
 
+        if clicked_marker is None:
+            return
+
         self.mapview.remove_marker(clicked_marker)
 
-        pothole_index = self.pothole_markers.index(clicked_marker)
-        bump_index = self.bump_markers.index(clicked_marker)
+        if clicked_marker in self.pothole_markers:
+            self.pothole_markers.pop(self.pothole_markers.index(clicked_marker))
+        elif clicked_marker in self.bump_markers:
+            self.bump_markers.pop(self.bump_markers.index(clicked_marker))
 
-        if pothole_marker >= 0:
-            self.pothole_markers.pop(pothole_index)
-        elif bump_index >= 0:
-            self.bump_markers.pop(bump_index)
+        self.datasource.update_db_record_visibility(clicked_marker_data.id)
 
     def on_touch_down(self, widget, touch):
         if touch.button == "right":
@@ -164,7 +170,6 @@ class MapViewApp(App):
             self.delete_pothole_marker(coordinate)
             return True
 
-
     def build(self):
         """
         Ініціалізує мапу MapView(zoom, lat, lon)
@@ -176,7 +181,7 @@ class MapViewApp(App):
             lon=30.5234
         )
 
-        self.mapview.bind(on_touch_down = self.on_touch_down)
+        self.mapview.bind(on_touch_down=self.on_touch_down)
 
         return self.mapview
 

store/Dockerfile-test

@@ -6,6 +6,7 @@ WORKDIR /app
 COPY store/requirements.txt .
 
 RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --root-user-action ignore --no-cache-dir pytest httpx
 # Copy the entire application into the container
 COPY store/. .
 # Run the main.py script inside the container when it starts

store/main.py

@@ -39,16 +39,21 @@ subscriptions: Set[WebSocket] = set()
 
 
 # FastAPI WebSocket endpoint
-@app.websocket("/ws/{user_id}")
-async def websocket_endpoint(websocket: WebSocket, user_id: int):
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
     await websocket.accept()
 
     subscriptions.add(websocket)
 
     try:
         # send already available data
-        r = processed_agent_data.select().order_by(processed_agent_data.c.timestamp)
-        stored_data = SessionLocal().execute(r).fetchall()
+        r = processed_agent_data.select() \
+                                .where(processed_agent_data.c.visible == True) \
+                                .order_by(processed_agent_data.c.timestamp)
+
+        session = SessionLocal()
+        stored_data = session.execute(r).fetchall()
+        session.close() 
 
         jsonable_data = [{c.name: getattr(i, c.name) for c in processed_agent_data.columns} for i in stored_data]
         for i in jsonable_data:
@@ -198,9 +203,13 @@ def update_processed_agent_data(processed_agent_data_id: int, data: ProcessedAge
         session.commit()
 
         updated_result = session.execute(query).fetchone()
-
         return ProcessedAgentDataInDB(**updated_result._mapping)
         
+    except Exception as err:
+        session.rollback()
+        print(f"Database error: {err}")
+        raise HTTPException(status_code=500, detail="Internal Server Error")
+
     finally:
         session.close()
 
@@ -231,6 +240,11 @@ def delete_processed_agent_data(processed_agent_data_id: int):
 
         return ProcessedAgentDataInDB(**result._mapping)
         
+    except Exception as err:
+        session.rollback()
+        print(f"Database error: {err}")
+        raise HTTPException(status_code=500, detail="Internal Server Error")
+        
     finally:
         session.close()
 

store/test-entry.sh

@@ -1,3 +1,4 @@
 #!/bin/sh
 
 PYTHONPATH=$PWD python3 test/main_test.py
+PYTHONPATH=$PWD python3 -m pytest test_db.py

store/test_db.py

@@ -0,0 +1,45 @@
+from fastapi.testclient import TestClient
+from main import app, processed_agent_data
+from database import SessionLocal
+from sqlalchemy import select
+from datetime import datetime
+
+client = TestClient(app)
+
+
+def test_create_processed_agent_data():
+    db = SessionLocal()
+    before = db.execute(select(processed_agent_data)).fetchall()
+    before_count = len(before)
+    payload = {
+        "user_id": 123,
+        "data": [
+            {
+                "road_state": "normal",
+                "agent_data": {
+                    "user_id": 999, 
+                    "accelerometer": {
+                        "x": 1.1,
+                        "y": 2.2,
+                        "z": 3.3
+                    },
+                    "gps": {
+                        "latitude": 50.45,
+                        "longitude": 30.52
+                    },
+                    "timestamp": datetime.now().isoformat()
+                }
+            }
+        ]
+    }
+    
+    response = client.post("/processed_agent_data/", json=payload)
+
+    assert response.status_code == 200
+
+    after = db.execute(select(processed_agent_data)).fetchall()
+    after_count = len(after)
+
+    assert after_count == before_count + 1
+
+    db.close()