Compare commits

...

25 Commits

Author SHA1 Message Date
a57097d504 Set default for archives that don't exist 2022-10-11 06:55:11 +01:00
2c7b0d2ced Parse proxy ip headers 2022-10-05 18:40:09 +01:00
efb4719914 Convert sizes to big integers 2022-10-05 18:27:38 +01:00
b03b4fadce Use remote cache 2022-10-05 16:51:51 +01:00
cde6c1d288 Create update script to be run as user 2022-10-05 16:48:11 +01:00
53568e360f Add logging 2022-10-05 16:28:46 +01:00
c81e018d97 Use postgres db and add proxy to allowed hosts 2022-10-05 16:27:51 +01:00
71d911c660 Log archive post failure 2022-10-05 16:24:05 +01:00
2fb284a142 Rename django cache to avoid conflicts 2022-10-05 16:21:20 +01:00
c79af4b675 Install postgres pip package 2022-10-05 08:07:34 +01:00
8c8465df52 Don't clear cache 2022-10-05 07:40:33 +01:00
fba957a6bc Allow for toggling of repo visibility 2022-09-28 16:56:44 +01:00
c506d9fcfa Add field to allow repo hiding 2022-07-26 15:25:27 +01:00
b4a1d99ae1 Update warning and error timings 2022-04-13 07:22:17 +01:00
02c10e0c78 Remove unused files 2022-04-13 02:02:20 +01:00
938be8c808 Implement basic error display 2022-04-13 01:33:09 +01:00
29d5b51752 Create graphs based on html data attribute 2022-04-11 11:30:15 +01:00
3031454682 Retrieve monthly archives more efficiently 2022-04-11 10:03:23 +01:00
aa4f31df23 Change database extension 2022-04-11 10:02:21 +01:00
1f5c2b9de6 Ignore database files 2022-04-11 10:01:55 +01:00
b743b7919c Call methods asynchronously 2022-04-11 09:04:16 +01:00
9ec5b178ac Use smaller loading spinners 2022-04-11 08:48:39 +01:00
1651b2f4ef Launch update methods asynchronously 2022-04-11 08:41:32 +01:00
b6f204a15e Use separate event methods 2022-04-11 08:30:58 +01:00
f16a0d5b65 Remove legend 2022-04-11 08:13:55 +01:00
24 changed files with 327 additions and 182 deletions

1
.gitignore vendored
View File

@ -35,6 +35,7 @@ borgweb/borgweb/secrets.py
# db
*.sqlite
*.db
# Byte-compiled / optimized / DLL files
__pycache__/

View File

@ -1,4 +1,4 @@
from .repoform import RepoForm
from .repoform import RepoForm, ToggleVisibility
from .archiveform import ArchiveForm
from .errorform import ErrorForm
from .locationform import LocationForm

View File

@ -6,3 +6,7 @@ class RepoForm(forms.Form):
fingerprint = forms.CharField(label='Fingerprint')
location = forms.CharField(label='Location')
last_modified = forms.DateTimeField(label='Last Modified', input_formats=["%Y-%m-%dT%H:%M:%S.%z"])
class ToggleVisibility(forms.Form):
label = forms.CharField(label='Label')

View File

@ -0,0 +1,18 @@
# Generated by Django 4.0.6 on 2022-07-26 15:24
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('borg', '0002_location'),
]
operations = [
migrations.AddField(
model_name='label',
name='visible',
field=models.BooleanField(default=True),
),
]

View File

@ -0,0 +1,63 @@
# Generated by Django 4.0.6 on 2022-10-05 18:27
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('borg', '0003_label_visible'),
]
operations = [
migrations.AlterField(
model_name='archive',
name='compressed_size',
field=models.BigIntegerField(),
),
migrations.AlterField(
model_name='archive',
name='deduplicated_size',
field=models.BigIntegerField(),
),
migrations.AlterField(
model_name='archive',
name='file_count',
field=models.BigIntegerField(),
),
migrations.AlterField(
model_name='archive',
name='original_size',
field=models.BigIntegerField(),
),
migrations.AlterField(
model_name='cache',
name='total_chunks',
field=models.BigIntegerField(),
),
migrations.AlterField(
model_name='cache',
name='total_csize',
field=models.BigIntegerField(),
),
migrations.AlterField(
model_name='cache',
name='total_size',
field=models.BigIntegerField(),
),
migrations.AlterField(
model_name='cache',
name='total_unique_chunks',
field=models.BigIntegerField(),
),
migrations.AlterField(
model_name='cache',
name='unique_csize',
field=models.BigIntegerField(),
),
migrations.AlterField(
model_name='cache',
name='unique_size',
field=models.BigIntegerField(),
),
]

View File

@ -8,9 +8,9 @@ class Archive(models.Model):
name = models.TextField()
start = models.DateTimeField()
end = models.DateTimeField()
file_count = models.IntegerField()
original_size = models.IntegerField()
compressed_size = models.IntegerField()
deduplicated_size = models.IntegerField()
file_count = models.BigIntegerField()
original_size = models.BigIntegerField()
compressed_size = models.BigIntegerField()
deduplicated_size = models.BigIntegerField()
cache = models.OneToOneField(Cache, on_delete=models.CASCADE)

View File

@ -2,9 +2,9 @@ from django.db import models
class Cache(models.Model):
total_chunks = models.IntegerField()
total_csize = models.IntegerField()
total_size = models.IntegerField()
total_unique_chunks = models.IntegerField()
unique_csize = models.IntegerField()
unique_size = models.IntegerField()
total_chunks = models.BigIntegerField()
total_csize = models.BigIntegerField()
total_size = models.BigIntegerField()
total_unique_chunks = models.BigIntegerField()
unique_csize = models.BigIntegerField()
unique_size = models.BigIntegerField()

View File

@ -3,6 +3,7 @@ from django.db import models
class Label(models.Model):
label = models.TextField(blank=True, unique=True)
visible = models.BooleanField(default=True)
def __str__(self):
return self.label

View File

@ -11,10 +11,10 @@ class Repo(models.Model):
last_modified = models.DateTimeField()
label = models.OneToOneField(Label, on_delete=models.CASCADE, unique=True)
def warning(self, hours_ago=2):
def warning(self, hours_ago=4):
return not self.latest_archive().start > datetime.utcnow() - timedelta(hours=hours_ago)
def error(self, hours_ago=4):
def error(self, hours_ago=12):
latest_archive = self.latest_archive()
if latest_archive is None or not self.archive_after_latest_error():
return True
@ -102,23 +102,23 @@ class Repo(models.Model):
@staticmethod
def series_csize(archives, units=None):
return [convert_bytes(archive.cache.unique_csize, units)[0]
if archive is not None else None for archive in archives]
if archive is not None else 0 for archive in archives]
def hourly_archive_string(self):
return ''.join(['H' if archive is not None else '-' for archive in self.hourly_archives(8)])
def monthly_archives(self, n_months: int = 12):
archives = []
for month in range(n_months):
current_date = subtract_months(datetime.utcnow(), month)
archive_current_month = self.archive_set.all() \
.filter(start__year=current_date.year,
start__month=current_date.month) \
.order_by('-start')
if len(archive_current_month) > 0:
archives.append(archive_current_month[0])
else:
archives.append(None)
archive_set = self.archive_set.all().order_by('-start')
current_date = datetime.utcnow()
for archive in archive_set:
if len(archives) >= n_months:
break
if archive.start.year == current_date.year and archive.start.month == current_date.month:
archives.append(archive)
current_date = subtract_months(current_date, 1)
return archives[::-1]
def archives_on_dates(self, dates: list):
@ -136,9 +136,9 @@ class Repo(models.Model):
archives = []
for hour in range(n_hours):
current_hour = datetime.utcnow() - timedelta(hours=hour)
archives_hour = self.archive_set.all()\
.filter(start__date=current_hour.date())\
.filter(start__hour=current_hour.hour)\
archives_hour = self.archive_set.all() \
.filter(start__date=current_hour.date()) \
.filter(start__hour=current_hour.hour) \
.order_by('-start')
if len(archives_hour) > 0:
archives.append(archives_hour[0])

View File

@ -1,19 +1,19 @@
function draw_time_series_graph(chartID, repo, dateLabels, sizeUnits) {
function draw_time_series_graph(canvas, data) {
let datasets = [{
label: repo.label,
data: repo.size,
label: data.label,
data: data.size,
fill: false,
borderColor: 'rgb(7, 59, 76)'
}]
const data = {
labels: dateLabels,
const graphData = {
labels: data.dates,
datasets: datasets
};
const config = {
type: 'line',
data,
data: graphData,
options: {
plugins: {
tooltip: {
@ -21,13 +21,16 @@ function draw_time_series_graph(chartID, repo, dateLabels, sizeUnits) {
label: function (context) {
const yValue = context.parsed.y
if (yValue !== null) {
return `${yValue} ${sizeUnits}`
return `${yValue} ${data.units}`
} else {
return ""
}
}
}
}
},
legend: {
display: false
},
},
scales: {
y: {
@ -41,7 +44,7 @@ function draw_time_series_graph(chartID, repo, dateLabels, sizeUnits) {
},
ticks: {
callback: function (value, index, values) {
return `${value} ${sizeUnits}`
return `${value} ${data.units}`
}
}
}
@ -49,69 +52,8 @@ function draw_time_series_graph(chartID, repo, dateLabels, sizeUnits) {
}
}
var newChart = new Chart(
document.getElementById(chartID),
config
);
}
function draw_time_graph(chartID, repos, dateLabels, sizeUnits) {
let datasets = []
repos.forEach(function (repo) {
datasets.push({
label: repo.label,
data: repo.size,
fill: false,
borderColor: 'rgb(7, 59, 76)'
});
})
const data = {
labels: dateLabels,
datasets: datasets
};
const config = {
type: 'line',
data,
options: {
plugins: {
tooltip: {
callbacks: {
label: function (context) {
const yValue = context.parsed.y
if (yValue !== null) {
return `${yValue} ${sizeUnits}`
} else {
return ""
}
}
}
}
},
scales: {
y: {
min: 0,
title: {
display: true,
text: "Compressed Size",
font: {
size: 18
}
},
ticks: {
callback: function (value, index, values) {
return `${value} ${sizeUnits}`
}
}
}
}
}
}
var newChart = new Chart(
document.getElementById(chartID),
const newGraph = new Chart(
canvas,
config
);
}

View File

@ -12,29 +12,37 @@ function colourRepo(repo_json, label, container_id) {
$(container_id).find(repoLabel).addClass(bg_class);
}
window.addEventListener("DOMContentLoaded", function () {
// todo: inflate each repo and colour background accordingly
const container = $('#repo-container');
function stringRequests() {
$('[data-json-string-request]').each(function (index, element) {
$.getJSON($(this).attr("data-json-string-request"), function (data) {
$(element).html(data['data']);
})
});
});
}
$.getJSON(`/repo-list.json`, function (repo_list) {
function graphRequests() {
$('[data-json-graph-request]').each(function (index, element) {
$.getJSON($(this).attr("data-json-graph-request"), function (data) {
let newGraph = $('<canvas/>').width(400).height(200);
draw_time_series_graph(newGraph, data)
$(element).html(newGraph);
});
});
}
function colourRepos(repo_list) {
const container = $('#repo-container');
repo_list.labels.forEach(function (repo_label) {
$.getJSON(`/repo/${repo_label}.json`, function (repo_json) {
colourRepo(repo_json, repo_label, container);
})
$.getJSON(`/repo/${repo_label}/monthly-size.json`, function (repo_size_json) {
draw_time_series_graph(`repo-${repo_label}-size-graph`, repo_size_json.repo,
repo_size_json.dates, repo_size_json.units);
})
});
})
});
}
window.addEventListener("DOMContentLoaded", function () {
setTimeout(stringRequests, 0);
setTimeout(graphRequests, 0);
$.getJSON(`/repo-list.json`, function (repo_list) {
setTimeout(colourRepos.bind(null, repo_list), 0);
});
}, false);

View File

@ -0,0 +1,35 @@
{% extends "borg/base.html" %}
{% load cache %}
{% load static %}
{% block title %}{{ repo.label }} errors{% endblock %}
{% block script %}
{% endblock %}
{% block style %}
{{ block.super }}
.error-container {
padding: 8px;
margin: 8px;
}
{% endblock %}
{% block body %}
{% if errors %}
<div class="error-container" class="grid justify-content-left">
<ul class="att-label row ps-3 col-11">
{% for error in errors %}
<li class="shadow rounded overflow-hidden bg-primary m-1">
<span>{{ error.error }}</span>
</li>
{% endfor %}
</ul>
{% else %}
<div style="width: 600px;" class="error-container shadow rounded bg-primary overflow-hidden">
<div style="width: 600px;" class="error-container bg-primary overflow-hidden">
<h2 class="h2">No errors found</h2>
</div>
</div>
{% endif %}
</div>
{% endblock %}

View File

@ -6,7 +6,6 @@
{% block script %}
<script src="https://cdn.jsdelivr.net/npm/chart.js@3.4.1/dist/chart.min.js"
integrity="sha256-GMN9UIJeUeOsn/Uq4xDheGItEeSpI5Hcfp/63GclDZk=" crossorigin="anonymous"></script>
{% include "borg/repo-template.html" %}
<script src="{% static 'borg/js/chart.js' %}"></script>
<script src="{% static 'borg/js/index.js' %}"></script>
{% endblock %}
@ -38,7 +37,7 @@
<dt class="col-4">Latest backup:</dt>
<dd class="repo-latest-backup col-8"
data-json-string-request="/repo/{{ repo.label }}/latest-backup.json">
<div class="spinner-border" role="status">
<div class="spinner-border spinner-border-sm" role="status">
</div>
</dd>
</dl>
@ -46,7 +45,7 @@
<dt class="col-4">Size:</dt>
<dd class="repo-size col-8"
data-json-string-request="/repo/{{ repo.label }}/size.json">
<div class="spinner-border" role="status">
<div class="spinner-border spinner-border-sm" role="status">
</div>
</dd>
</dl>
@ -54,12 +53,16 @@
<dt class="col-4">Recent errors:</dt>
<dd class="repo-recent-errors col-8"
data-json-string-request="/repo/{{ repo.label }}/recent-errors.json">
<div class="spinner-border" role="status">
<div class="spinner-border spinner-border-sm" role="status">
</div>
</dd>
</dl>
<canvas id="repo-{{ repo.label }}-size-graph" width="400" height="200"></canvas>
</dl>
<div id="repo-{{ repo.label }}-size-graph"
class="d-flex justify-content-center"
data-json-graph-request="/repo/{{ repo.label }}/monthly-size.json">
<div class="spinner-border" role="status">
</div>
</div>
</div>
{% endfor %}
{% else %}

View File

@ -0,0 +1,14 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Toggle repo visibility</title>
</head>
<body>
<form action="toggle" method="post">
{% csrf_token %}
{{ form }}
<input type="submit" value="Submit">
</form>
</body>
</html>

View File

@ -1,23 +0,0 @@
<script id="repo-template" type="text/x-custom-template">
<div style="width: 600px;" class="repo-container shadow rounded overflow-hidden">
<div class="row me-1 overflow-hidden text-truncate">
<h2 class="h2"> <span class="repo-label"></span>
<small class="repo-location text-muted"></small>
</h2>
</div>
<dl class="att-label row ps-3">
<dt class="col-4">Latest backup:</dt>
<dd class="repo-latest-backup col-8"></dd>
</dl>
<dl class="att-label row ps-3">
<dt class="col-4">Size:</dt>
<dd class="repo-size col-8"></dd>
</dl>
<dl class="att-label row ps-3">
<dt class="col-4">Recent errors:</dt>
<dd class="repo-recent-errors col-8"></dd>
</dl>
<canvas class="repo-size-graph" width="400" height="200"></canvas>
</dl>
</div>
</script>

View File

@ -10,7 +10,7 @@ urlpatterns = [
path('repo-list.json', cache_page(60)(views.repo_list_json), name='repo list'),
# Repo
path('repo/<str:repo_label>/monthly-size.json', cache_page(3600)(views.repo_monthly_size_json),
path('repo/<str:repo_label>/monthly-size.json', cache_page(60)(views.repo_monthly_size_json),
name='repo size time series'),
path('repo/<str:repo_label>.json', cache_page(60)(views.repo_json), name='repo json'),
path('repo/<str:repo_label>/latest-backup.json', cache_page(60)(views.repo_latest_backup_json), name='repo json'),
@ -20,9 +20,11 @@ urlpatterns = [
# Repo page
path('repo/<str:repo_label>', cache_page(60)(views.repo), name='repo'),
path('repo/<str:repo_label>/errors', cache_page(60)(views.repo_errors), name='repo'),
# POST
path('post/repo', views.post_repo, name='post repo'),
path('post/toggle', views.toggle_visibility, name='toggle repo visibility'),
path('post/archive', views.post_archive, name='post archive'),
path('post/error', views.post_error, name='post error'),
path('post/location', views.post_location, name='post location'),

View File

@ -35,14 +35,10 @@ def repo_monthly_size_json(request, repo_label, months_ago: int = 12):
max_unit = get_units([repo])
repo_dict = {"id": repo.id,
"label": repo.label.label,
"size": repo.size_on_months(max_unit, months_ago)}
response_dict = {
"dates": date_labels,
"repo": repo_dict,
"units": max_unit
"units": max_unit,
"size": repo.size_on_months(max_unit, months_ago)
}
return JsonResponse(response_dict)

View File

@ -3,9 +3,34 @@ from django.http import HttpResponseRedirect
from django.urls import reverse
from django.contrib.auth.decorators import permission_required
from django.core.cache import cache
from django.core.cache import cache as django_cache
from ..models import Repo, Label, Archive, Cache, Error, Location
from ..forms import RepoForm, ArchiveForm, ErrorForm, LocationForm
from ..forms import RepoForm, ArchiveForm, ErrorForm, LocationForm, ToggleVisibility
import logging
logger = logging.getLogger(__file__)
@permission_required("borg.change_repo")
def toggle_visibility(request):
if request.method == 'POST':
form = ToggleVisibility(request.POST)
if form.is_valid():
cdata = form.cleaned_data
label = get_object_or_404(Label, label=cdata['label'])
label.visible = not label.visible
label.save()
django_cache.clear()
return HttpResponseRedirect(reverse('index'))
else:
form = ToggleVisibility()
return render(request, 'borg/post/toggle.html', {'form': form})
@permission_required("borg.add_repo")
@ -28,7 +53,7 @@ def post_repo(request):
'last_modified': cdata['last_modified'],
'label': label})
repo.save()
cache.clear()
django_cache.clear()
return HttpResponseRedirect(reverse('index'))
else:
@ -42,6 +67,7 @@ def post_archive(request):
if request.method == 'POST':
form = ArchiveForm(request.POST)
if form.is_valid():
try:
cdata = form.cleaned_data
repo = get_object_or_404(Repo, label__label=cdata['label'])
@ -57,7 +83,9 @@ def post_archive(request):
archive = Archive(**archive_dict, repo=repo, cache=cache)
archive.save()
cache.clear()
django_cache.clear()
except Exception:
logger.exception("Archive post failed")
return HttpResponseRedirect(reverse('index'))
else:
@ -76,7 +104,7 @@ def post_error(request):
error = Error(label=label, error=cdata['error'], time=cdata['time'])
error.save()
cache.clear()
django_cache.clear()
return HttpResponseRedirect(reverse('index'))
else:
@ -94,7 +122,7 @@ def post_location(request):
label, _ = Location.objects.get_or_create(label=cdata['label'],
defaults={"path": cdata["path"]})
label.save()
cache.clear()
django_cache.clear()
return HttpResponseRedirect(reverse('index'))
else:

View File

@ -1,9 +1,10 @@
from django.contrib.auth.decorators import permission_required
from django.shortcuts import render, get_object_or_404
from ..models import Repo
def index(request):
repo_list = Repo.objects.all()
repo_list = Repo.objects.filter(label__visible=True)
context = {
'repo_list': repo_list,
@ -16,5 +17,11 @@ def repo(request, repo_label: str):
return render(request, 'borg/repo.html', {'repo': s_repo})
@permission_required("borg.view_error")
def repo_errors(request, repo_label: str):
s_repo = get_object_or_404(Repo, label__label=repo_label)
return render(request, 'borg/errors.html', {'errors': s_repo.label.errors.all().order_by('-time')})
def axes(request, credentials, *args, **kwargs):
return render(request, 'error/axes.html', {})

View File

@ -1 +1 @@
from .secrets import SECRET_KEY
from .secrets import SECRET_KEY, DATABASE_PASSWORD

View File

@ -2,6 +2,7 @@ import os
from pathlib import Path
from . import SECRET_KEY as __SECRET_KEY
from . import DATABASE_PASSWORD as __DATABASE_PASSWORD
BASE_DIR = Path(__file__).resolve().parent.parent
@ -9,9 +10,11 @@ SECRET_KEY = __SECRET_KEY
DEBUG = False
AXES_META_PRECEDENCE_ORDER = ('HTTP_X_FORWARDED_FOR', 'X_FORWARDED_FOR', 'REMOTE_ADDR')
AXES_LOCKOUT_CALLABLE = "borg.views.axes"
ALLOWED_HOSTS = ['127.0.0.1', 'borg.george.ooo', 'george.ooo', 'www.george.ooo']
ALLOWED_HOSTS = ['127.0.0.1', 'borg.george.ooo', 'george.ooo', 'www.george.ooo', '10.10.10.100', 'proxy.george.ooo']
AUTHENTICATION_BACKENDS = [
'axes.backends.AxesBackend',
@ -65,8 +68,12 @@ WSGI_APPLICATION = 'borgweb.wsgi.application'
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': "/home/web/sites/db/borg.db",
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': 'borgweb',
'USER': 'borgweb',
'PASSWORD': __DATABASE_PASSWORD,
'HOST': 'db.george.ooo',
'PORT': '5432',
}
}
@ -120,7 +127,7 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://127.0.0.1:6379/1",
"LOCATION": "redis://cache.george.ooo:6379/1",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient"
},
@ -128,6 +135,36 @@ CACHES = {
}
}
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"root": {"level": "INFO", "handlers": ["file"]},
"handlers": {
"file": {
"level": "INFO",
"class": "logging.FileHandler",
"filename": "/var/log/django/borgweb.log",
"formatter": "app",
},
},
"loggers": {
"django": {
"handlers": ["file"],
"level": "INFO",
"propagate": True
},
},
"formatters": {
"app": {
"format": (
u"%(asctime)s [%(levelname)-8s] "
"(%(module)s.%(funcName)s) %(message)s"
),
"datefmt": "%Y-%m-%d %H:%M:%S",
},
},
}
# security
SESSION_COOKIE_SECURE = True
SECURE_SSL_REDIRECT = True

View File

@ -80,7 +80,7 @@ WSGI_APPLICATION = 'borgweb.wsgi.application'
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'borg.sqlite',
'NAME': BASE_DIR / 'borg.db',
}
}

View File

@ -35,7 +35,7 @@ print_action "Installing pip packages, this may take a while..."
# install required pip packages
yes | python -m pip install --upgrade wheel
yes | python -m pip install django gunicorn django-libsass django-compressor django-axes django-redis
yes | python -m pip install django gunicorn django-libsass django-compressor django-axes django-redis psycopg2-binary
print_action "Setting up static files and database"

9
borgweb/update.sh Normal file
View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
cd "${0%/*}"
source ./venv/bin/activate
python ./manage.py collectstatic --noinput
python ./manage.py compress
python ./manage.py migrate --noinput
deactivate