From bb2e997705d6a1c4cd123b633e32d6abd441d0b9 Mon Sep 17 00:00:00 2001 From: George Lacey Date: Tue, 4 May 2021 04:18:42 +0100 Subject: [PATCH] Parse borg JSON instead of raw output - Create db connection interface - Create repo and archive database classes --- .gitignore | 3 ++ src/database.py | 32 ------------- src/database/__init__.py | 3 ++ src/database/archive.py | 48 +++++++++++++++++++ src/database/databaseconnection.py | 77 ++++++++++++++++++++++++++++++ src/database/repo.py | 51 ++++++++++++++++++++ src/logentry.py | 58 ---------------------- src/main.py | 49 +++++++------------ 8 files changed, 200 insertions(+), 121 deletions(-) delete mode 100644 src/database.py create mode 100644 src/database/__init__.py create mode 100644 src/database/archive.py create mode 100644 src/database/databaseconnection.py create mode 100644 src/database/repo.py delete mode 100644 src/logentry.py diff --git a/.gitignore b/.gitignore index 02cd516..94070a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# db files +*.sqlite + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/src/database.py b/src/database.py deleted file mode 100644 index 2c184c8..0000000 --- a/src/database.py +++ /dev/null @@ -1,32 +0,0 @@ -from logentry import LogEntry -import sqlite3 - - -class Database(object): - def __init__(self, path): - self.conn = sqlite3.connect(path) - self.table_name = "log" - self.create_log_table() - - def __del__(self): - self.conn.close() - - def commit(self): - self.conn.commit() - - def create_log_table(self): - query = f"""CREATE TABLE IF NOT EXISTS {self.table_name} ( - logID integer PRIMARY KEY, - name text NOT NULL, - fingerprint text NOT NULL, - start text NOT NULL, - end text NOT NULL, - filecount long NOT NULL);""" - self.conn.execute(query) - self.commit() - - def insert(self, log_entry: LogEntry): - query = f"INSERT INTO {self.table_name} (name, fingerprint, start, end, filecount) VALUES(?,?,?,?,?)" - self.conn.execute(query, (log_entry.name, log_entry.fingerprint, log_entry.start_time, log_entry.end_time, - log_entry.file_count)) - self.commit() diff --git a/src/database/__init__.py b/src/database/__init__.py new file mode 100644 index 0000000..8982607 --- /dev/null +++ b/src/database/__init__.py @@ -0,0 +1,3 @@ +from .databaseconnection import DatabaseConnection +from .repo import Repo +from .archive import Archive diff --git a/src/database/archive.py b/src/database/archive.py new file mode 100644 index 0000000..73a3686 --- /dev/null +++ b/src/database/archive.py @@ -0,0 +1,48 @@ +from . import DatabaseConnection, Repo +from datetime import datetime + + +class Archive(DatabaseConnection): + def __init__(self, db_path, repo: Repo, archive_json: dict, table_name: str = "archive"): + super().__init__(db_path, table_name) + + self.uuid = archive_json['id'] + self.repo_id = repo.repo_id + self.name = archive_json['name'] + print(archive_json['start']) + self.start = datetime.fromisoformat(archive_json['start']) + self.end = datetime.fromisoformat(archive_json['end']) + + self.archive_id = self._insert() + + def _create_table(self): + create_statement = f"create table if not exists {self._sql_table}(" \ + f"archive_id INTEGER PRIMARY KEY," \ + f"uuid INTEGER NOT NULL UNIQUE," \ + f"repo_id INTEGER NOT NULL," \ + f"name TEXT NOT NULL UNIQUE," \ + f"start TEXT TIMESTAMP NULL," \ + f"end TEXT TIMESTAMP NULL," \ + f"FOREIGN KEY (repo_id) REFERENCES repo (repo_id))" + self.sql_execute(create_statement) + + def _exists(self): + result = self.sql_execute_one(f"SELECT archive_id FROM {self._sql_table}" + f" WHERE uuid=?;", (self.uuid,)) + if result is None: + return None + else: + return result[0] + + def _insert(self) -> int: + if self._exists(): + raise Exception("archive with same uuid already exists") + with self.sql_lock: + cursor = self.sql_cursor + statement = f"INSERT INTO {self._sql_table}"\ + f" ('uuid', 'repo_id', 'name', 'start', 'end')"\ + f" VALUES (?, ?, ?, ?, ?);" + args = (self.uuid, self.repo_id, self.name, self.start, self.end) + cursor.execute(statement, args) + self.sql_commit() + return cursor.lastrowid diff --git a/src/database/databaseconnection.py b/src/database/databaseconnection.py new file mode 100644 index 0000000..c3cf21d --- /dev/null +++ b/src/database/databaseconnection.py @@ -0,0 +1,77 @@ +from abc import ABC, abstractmethod +from threading import Lock +import sqlite3 + + +class DatabaseConnection(ABC): + def __init__(self, db_path, table_name: str): + self.__sql_lock = Lock() + + self.__sql_database = sqlite3.connect(db_path, + detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES, + check_same_thread=False) + + self._sql_table = table_name + + self._create_table() + self.sql_commit() + + @property + def sql_lock(self): + return self.__sql_lock + + @property + def sql_cursor(self): + return self.__sql_database.cursor() + + def sql_execute(self, statement: str, args: tuple = None): + with self.__sql_lock: + cursor = self.sql_cursor + if args is None: + cursor.execute(statement) + else: + cursor.execute(statement, args) + + def sql_execute_row_id(self, statement: str, args: tuple = None): + with self.__sql_lock: + cursor = self.sql_cursor + if args is None: + cursor.execute(statement) + else: + cursor.execute(statement, args) + cursor.execute(f"select last_insert_rowid() from {self._sql_table};") + row_id = cursor.fetchone() + return row_id[0] + + def sql_execute_all(self, statement: str, args: tuple = None): + with self.__sql_lock: + cursor = self.sql_cursor + if args is None: + cursor.execute(statement) + else: + cursor.execute(statement, args) + + return cursor.fetchall() + + def sql_execute_one(self, statement: str, args: tuple = None): + with self.__sql_lock: + cursor = self.sql_cursor + if args is None: + cursor.execute(statement) + else: + cursor.execute(statement, args) + + return cursor.fetchone() + + def sql_commit(self): + self.__sql_database.commit() + + @abstractmethod + def _create_table(self): + raise NotImplementedError + + def stop(self): + with self.__sql_lock: + self.sql_commit() + self.__sql_database.close() + diff --git a/src/database/repo.py b/src/database/repo.py new file mode 100644 index 0000000..a43452e --- /dev/null +++ b/src/database/repo.py @@ -0,0 +1,51 @@ +from . import DatabaseConnection +from datetime import datetime + + +class Repo(DatabaseConnection): + def __init__(self, db_path, repo_json: dict, table_name: str = 'repo'): + super(Repo, self).__init__(db_path, table_name) + + self.repo_id = None + self.uuid = repo_json['id'] + self.location = repo_json['location'] + self.last_modified = datetime.fromisoformat(repo_json['last_modified']) + + repo_id = self._exists() + if repo_id is None: + self.repo_id = self._insert() + else: + self.repo_id = repo_id + self._update() + + def _insert(self) -> int: + with self.sql_lock: + cursor = self.sql_cursor + statement = f"INSERT INTO {self._sql_table}"\ + f" ('uuid', 'location', 'last_modified')"\ + f" VALUES (?, ?, ?);" + args = (self.uuid, self.location, self.last_modified) + cursor.execute(statement, args) + self.sql_commit() + return cursor.lastrowid + + def _update(self): + self.sql_execute(f"UPDATE {self._sql_table} SET location = ?, last_modified = ? WHERE repo_id = ?;", + (self.location, self.last_modified, self.repo_id)) + self.sql_commit() + + def _exists(self): + result = self.sql_execute_one(f"SELECT repo_id FROM {self._sql_table}" + f" WHERE uuid=?;", (self.uuid,)) + if result is None: + return None + else: + return result[0] + + def _create_table(self): + create_statement = f"create table if not exists {self._sql_table}(" \ + f"repo_id INTEGER PRIMARY KEY," \ + f"uuid INTEGER NOT NULL UNIQUE," \ + f"location TEXT NOT NULL," \ + f"last_modified TIMESTAMP NOT NULL)" + self.sql_execute(create_statement) diff --git a/src/logentry.py b/src/logentry.py deleted file mode 100644 index a151db3..0000000 --- a/src/logentry.py +++ /dev/null @@ -1,58 +0,0 @@ -from datetime import datetime -from math import floor -import re - - -class LogEntry(object): - - def __init__(self, name, fingerprint, start_time, end_time, duration_string, - file_count): - self.name = name - self.fingerprint = fingerprint - self.start_time = self.get_datetime(start_time) - self.end_time = self.get_datetime(end_time) - self.duration = self.get_duration(duration_string) - self.file_count = file_count - - def print_to_file(self, filename): - with open(filename, "w") as file: - file.writelines([f"name: {self.name}", - f"fingerprint: {self.fingerprint}", - f"start: {self.start_time.date()} time: {self.start_time.time()}", - f"end: {self.end_time.date()} time: {self.end_time.time()}", - f"duration: {self.duration}", - f"file_count: {self.file_count}"]) - - def get_duration(self, duration_string): - total_seconds = 0 - time_strings = [('second', 1), ('minute', 60), ('hour', 3600), ('day', 86400)] - for ts, mult in time_strings: - total_seconds += self.get_time_unit_string(duration_string, ts, mult) - - return floor(total_seconds) - - @staticmethod - def get_time_unit_string(text: str, time_text: str, multiplier: int = 1): - substring = re.search(rf"((\d+)\.(\d+)|(\d+))\s({time_text}|{time_text}s)", text) - if substring is not None: - substring = substring.group().strip(f" {time_text}s") - return float(substring) * multiplier - else: - return 0 - - @staticmethod - def get_datetime(datetime_string): - date_string = re.search(r"....-..-..", datetime_string).group() - time_string = re.search(r"..:..:..", datetime_string).group() - - year = int(date_string[0:4]) - month = int(date_string[5:7]) - day = int(date_string[8:10]) - - hour = int(time_string[0:2]) - minute = int(time_string[3:5]) - second = int(time_string[6:8]) - - converted_datetime = datetime(year, month, day, hour, minute, second) - - return converted_datetime diff --git a/src/main.py b/src/main.py index 12f4956..4b7206a 100644 --- a/src/main.py +++ b/src/main.py @@ -1,39 +1,26 @@ from sys import stdin -from src.logentry import LogEntry -from src.database import Database +from os.path import realpath +from pathlib import Path +from database import Repo, Archive +import json -def main(input_lines: list): - raw_borg_output = input_lines +def main(input_json: dict, path: Path): + db_path = path / 'borg.sqlite' - borg_log_entry = create_log_entry(raw_borg_output) - borg_log_entry.print_to_file("borg.txt") - - database = Database("borg.db") - database.insert(borg_log_entry) - - -def create_log_entry(raw_borg_output: list): - attributes = {"Archive name: ": "", - "Archive fingerprint: ": "", - "Time (start): ": "", - "Time (end): ": "", - "Duration: ": "", - "Number of files: ": ""} - - for line in raw_borg_output: - for key in attributes: - if line.startswith(key): - attributes[key] = line[len(key):].strip() - - return LogEntry(attributes["Archive name: "], - attributes["Archive fingerprint: "], - attributes["Time (start): "], - attributes["Time (end): "], - attributes["Duration: "], - attributes["Number of files: "]) + repo = Repo(db_path, input_json['repository']) + log_entry = Archive(db_path, repo, input_json['archive']) if __name__ == "__main__": + path = Path(realpath(__file__)).parent.parent input_text = stdin.readlines() - main(input_text) + + try: + input_json = json.loads(" ".join(input_text)) + except: + # todo: output input_text somewhere + print("Error parsing json, output:") + print("\n".join(input_text)) + exit(1) + main(input_json, path)