From c101aa8aa61b5f120668d836fb07e8be14b3db18 Mon Sep 17 00:00:00 2001 From: StNicolay Date: Sun, 11 Aug 2024 08:16:30 +0300 Subject: [PATCH] File sync --- .gitignore | 2 + desktop_client/auth.py | 12 +--- desktop_client/file_widgets.py | 85 +++++++++++++++++++----- desktop_client/folder_info.py | 37 +++++------ desktop_client/state.py | 7 ++ desktop_client/sync.py | 117 +++++++++++++++++++++++++++++++++ desktop_client/user.py | 4 +- 7 files changed, 214 insertions(+), 50 deletions(-) create mode 100644 desktop_client/sync.py diff --git a/.gitignore b/.gitignore index 489bdd0..6d7ebab 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +database.db diff --git a/desktop_client/auth.py b/desktop_client/auth.py index 167cdb9..9154f59 100644 --- a/desktop_client/auth.py +++ b/desktop_client/auth.py @@ -116,9 +116,7 @@ class LoginWidget(QWidget): password = self.password_input.text() if not username or not password: - QMessageBox.warning( - self, "Input Error", "Email and Password are required" - ) + QMessageBox.warning(self, "Input Error", "Email and Password are required") return try: @@ -131,12 +129,8 @@ class LoginWidget(QWidget): if access_token: self.switcher.login(access_token) else: - QMessageBox.warning( - self, "Error", "No access token received" - ) + QMessageBox.warning(self, "Error", "No access token received") else: - QMessageBox.warning( - self, "Error", f"Login failed: {response.text}" - ) + QMessageBox.warning(self, "Error", f"Login failed: {response.text}") except httpx.HTTPError as e: QMessageBox.critical(self, "HTTP Error", str(e)) diff --git a/desktop_client/file_widgets.py b/desktop_client/file_widgets.py index 94003c9..ea3f959 100644 --- a/desktop_client/file_widgets.py +++ b/desktop_client/file_widgets.py @@ -1,7 +1,9 @@ from __future__ import annotations +import base64 import dataclasses import datetime +import hashlib import os import uuid from typing import Protocol, Self @@ -13,13 +15,7 @@ import pydantic import state import user from PyQt6.QtCore import QPoint, Qt -from PyQt6.QtGui import ( - QAction, - QDragEnterEvent, - QDragMoveEvent, - QDropEvent, - QIcon, -) +from PyQt6.QtGui import QAction, QDragEnterEvent, QDragMoveEvent, QDropEvent, QIcon from PyQt6.QtWidgets import ( QFileDialog, QHBoxLayout, @@ -64,9 +60,7 @@ class File(pydantic.BaseModel): return self.file_name def delete(self) -> None: - RequestClient().client.delete( - "/files", params={"file_id": self.file_id} - ) + RequestClient().client.delete("/files", params={"file_id": self.file_id}) def details(self, list: FileListWidget) -> QWidget: del list @@ -112,6 +106,56 @@ class File(pydantic.BaseModel): for data in stream.iter_bytes(): f.write(data) + def create(path: str, parent_id: uuid.UUID): + """Upload the file""" + file_name = os.path.basename(path) + try: + with open(path, "rb") as f: + files = {"file": (file_name, f)} + response = RequestClient().client.post( + "/files", + files=files, + params={"parent_folder": parent_id}, + ) + if not response.is_success: + QMessageBox.warning( + None, "Error", f"Upload failed: {response.text}" + ) + print(response.text) + except httpx.HTTPError as e: + QMessageBox.critical(None, "HTTP Error", str(e)) + + def download(self, path: str): + with open(path, "wb") as f, RequestClient().client.stream( + "GET", "/files", params={"file_id": self.file_id} + ) as stream: + if not stream.is_success: + QMessageBox.warning(None, "Error downloading the file") + return + for data in stream.iter_bytes(): + f.write(data) + + def modify(self, path: str): + """Upload the file""" + file_name = os.path.basename(path) + hash = hashlib.file_digest(open(path, "rb"), "sha512").digest() + if base64.b64decode(self.sha512) == hash: + return + try: + with open(path, "rb") as f: + files = {"file": (file_name, f)} + response = RequestClient().client.patch( + "/files", + files=files, + params={"file_id": self.file_id}, + ) + if not response.is_success: + QMessageBox.warning( + None, "Error", f"Upload failed: {response.text}" + ) + except httpx.HTTPError as e: + QMessageBox.critical(None, "HTTP Error", str(e)) + class Folder(pydantic.BaseModel): folder_id: uuid.UUID @@ -139,6 +183,19 @@ class Folder(pydantic.BaseModel): list.responses.append(ListResponse.get(self.folder_id)) list.update() + @staticmethod + def create(name: str, parent_id: uuid.UUID) -> uuid.UUID: + response = RequestClient().client.post( + "/folders", + json={ + "folder_name": name, + "parent_folder_id": str(parent_id), + }, + ) + if not response.is_success: + QMessageBox.warning(None, "Error creating folder", response.text) + return uuid.UUID(response.text.strip('"')) + class ResponseProtocol(Protocol): def items(self) -> list[DisplayProtocol]: @@ -172,9 +229,7 @@ class ListResponse(pydantic.BaseModel): return self.get(self.folder_id) def create_folder(self, file_list: FileListWidget): - return create_folder_widget.CreateFolderWidget( - self.folder_id, file_list - ) + return create_folder_widget.CreateFolderWidget(self.folder_id, file_list) @dataclasses.dataclass(slots=True) @@ -250,9 +305,7 @@ class FileListWidget(QListWidget): response = RequestClient().client.post( "http://localhost:3000/files", files=files, - params={ - "parent_folder": self.current_response().folder_id - }, + params={"parent_folder": self.current_response().folder_id}, ) if response.is_success: QMessageBox.information( diff --git a/desktop_client/folder_info.py b/desktop_client/folder_info.py index f8e3da3..feca743 100644 --- a/desktop_client/folder_info.py +++ b/desktop_client/folder_info.py @@ -1,23 +1,24 @@ from __future__ import annotations +import enum import uuid from dataclasses import dataclass -from request_client import RequestClient -import enum +from functools import cache, partial + import file_widgets +import pydantic +import user from PyQt6.QtWidgets import ( - QWidget, - QMessageBox, - QLabel, - QPushButton, QComboBox, QHBoxLayout, - QVBoxLayout, + QLabel, QLineEdit, + QMessageBox, + QPushButton, + QVBoxLayout, + QWidget, ) -import pydantic -from functools import cache, partial -import user +from request_client import RequestClient class Permission(enum.StrEnum): @@ -35,9 +36,7 @@ class Permission(enum.StrEnum): }, ) if not response.is_success: - QMessageBox.warning( - None, "Error setting permissions", response.text - ) + QMessageBox.warning(None, "Error setting permissions", response.text) @staticmethod def delete(folder_id: uuid.UUID, user_id: int, widget: FolderInfoWidget): @@ -49,9 +48,7 @@ class Permission(enum.StrEnum): }, ) if not response.is_success: - QMessageBox.warning( - None, "Error deleting permissions", response.text - ) + QMessageBox.warning(None, "Error deleting permissions", response.text) else: widget.redraw() @@ -97,14 +94,10 @@ class FolderInfoWidget(QWidget): combo = QComboBox() combo.addItems(map(str, Permission)) combo.setCurrentText(str(permissions)) - combo.currentTextChanged.connect( - partial(self.change, user_id=user_id) - ) + combo.currentTextChanged.connect(partial(self.change, user_id=user_id)) delete = QPushButton("Delete") delete.clicked.connect( - partial( - Permission.delete, self.folder.folder_id, user_id, self - ) + partial(Permission.delete, self.folder.folder_id, user_id, self) ) layout.addWidget(name) layout.addWidget(combo) diff --git a/desktop_client/state.py b/desktop_client/state.py index 9bd5c10..3fca830 100644 --- a/desktop_client/state.py +++ b/desktop_client/state.py @@ -1,8 +1,11 @@ +import uuid + import auth import file_widgets import httpx import keyring import request_client +import sync from PyQt6.QtWidgets import QMainWindow, QStackedWidget @@ -39,6 +42,10 @@ class State(QMainWindow): def login(self, token: str): keyring.set_password("auth_app", "access_token", token) request_client.RequestClient().set_token(token) + sync.SyncData.sync_all( + # "/home/stnicolay/backups", + # uuid.UUID("0191397f-ae77-7b2a-bed7-9d28ed56a90a"), + ) self.file_widget = file_widgets.MainFileWidget(self) self.stack.addWidget(self.file_widget) self.stack.setCurrentWidget(self.file_widget) diff --git a/desktop_client/sync.py b/desktop_client/sync.py new file mode 100644 index 0000000..5924c85 --- /dev/null +++ b/desktop_client/sync.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import datetime +import os +import uuid +from concurrent.futures import ThreadPoolExecutor +from time import ctime + +import file_widgets +from request_client import RequestClient +from sqlmodel import Field, Session, SQLModel, create_engine, select + + +class FolderStructure(file_widgets.Folder): + files: list[file_widgets.File] + folders: list[FolderStructure] + + def find_folder(self, name: str) -> file_widgets.File | None: + for file in self.folders: + if file.folder_name == name: + return file + + def find_file(self, name: str) -> file_widgets.File | None: + for file in self.files: + if file.file_name == name: + return file + + @staticmethod + def get_structure(folder_id: uuid.UUID) -> FolderStructure: + return FolderStructure.model_validate_json( + RequestClient() + .client.get("/folders/structure", params={"folder_id": folder_id}) + .text + ) + + +class SyncData(SQLModel, table=True): + id: int | None = Field(default=None, primary_key=True) + path: str + last_updated: datetime.datetime + folder_id: uuid.UUID + + @staticmethod + def sync_all(): + with Session(engine) as s: + syncs = s.exec(select(SyncData)).fetchall() + with ThreadPoolExecutor(3) as pool: + map_ = pool.map( + lambda sync: merge_folder( + sync.path, + FolderStructure.get_structure(sync.folder_id), + sync.last_updated, + ), + syncs, + ) + tuple(map_) + + @staticmethod + def new_sync(path: str, parent_id): + name = os.path.basename(path) + folder_id = file_widgets.Folder.create(name, parent_id) + with Session(engine) as s: + sync = SyncData( + path=path, + last_updated=datetime.datetime.now(), + folder_id=folder_id, + ) + s.add(sync) + s.commit() + upload_folder(path, folder_id) + + +sqlite_file_name = "database.db" +sqlite_url = f"sqlite:///{sqlite_file_name}" + +engine = create_engine(sqlite_url, echo=True) +SQLModel.metadata.create_all(engine) + + +def upload_folder(path: str, folder_id: uuid.UUID): + items = os.listdir(path) + for item in items: + item_path = os.path.join(path, item) + if os.path.isfile(item_path): + file_widgets.File.create(item_path, folder_id) + elif os.path.isdir(item_path): + id = file_widgets.Folder.create(item, folder_id) + upload_folder(item_path, id) + + +def merge_folder( + path: str, + structure: FolderStructure, + time: datetime.datetime, +): + for item in os.listdir(path): + item_path = os.path.join(path, item) + if os.path.isfile(item_path): + file = structure.find_file(item) + if file is None: + file_widgets.File.create(item_path, structure.folder_id) + continue + mtime = datetime.datetime.strptime( + ctime(os.path.getmtime(item_path)), "%a %b %d %H:%M:%S %Y" + ) + print(item, mtime, time) + if mtime > time: + file.modify(item_path) + else: + file.download(item_path) + elif os.path.isdir(item_path): + folder = structure.find_folder(item) + if folder is None: + folder_id = file_widgets.Folder.create(item, structure.folder_id) + upload_folder(item_path, folder_id) + else: + merge_folder(item_path, folder, time) diff --git a/desktop_client/user.py b/desktop_client/user.py index 317fe7e..1f2ed0e 100644 --- a/desktop_client/user.py +++ b/desktop_client/user.py @@ -29,9 +29,7 @@ class User(pydantic.BaseModel): url = "/users/current" response = RequestClient().client.get(url, params=params) if not response.is_success: - QMessageBox.warning( - None, "Error getting permissions", response.text - ) + QMessageBox.warning(None, "Error getting permissions", response.text) return return User.model_validate_json(response.text)