Implementing a Robust SQLite Backup System in Django
There here many ways to backup database & here we are storing backup file in GitHub for simplicity but you can modify the script & push it elsewhere.
In this blog post, I'll share our approach to implementing a comprehensive SQLite backup system for a Django application. This system not only creates backups but also verifies their integrity and automatically pushes them to GitHub for safe storage.
we will use sqlite online backup API for this. you can read more about it here https://www.sqlite.org/backup.html
The Online Backup API was created to address these concerns. The online backup API allows the contents of one database to be copied into another database file, replacing any original contents of the target database. The copy operation may be done incrementally, in which case the source database does not need to be locked for the duration of the copy, only for the brief periods of time when it is actually being read from. This allows other database users to continue without excessive delays while a backup of an online database is made.
System Components
1. Backup Creation
Uses SQLite's Online Backup API
Creates timestamped backups while the database is in use
Maintains atomic consistency during backup
Stores backups in a configured directory
apps/realtime_timer/management/commands/backup_database.py
import os
import sqlite3
from datetime import datetime
from django.core.management.base import BaseCommand
from django.conf import settings
import logging
logger = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Django management command to create SQLite database backups.
Uses SQLite's built-in online backup API which allows creating backups
while the database is in use.
Usage:
python manage.py backup_database
The backup will be stored in the BACKUP_DIRECTORY specified in settings.SQLITE_BACKUP
with a timestamp in the filename.
"""
help = 'Creates a backup of the SQLite database using the online backup API'
def handle(self, *args, **options):
try:
# Create backup directory if it doesn't exist
# Uses the path specified in settings.SQLITE_BACKUP['BACKUP_DIRECTORY']
backup_dir = os.path.join(settings.BASE_DIR, 'backups')
os.makedirs(backup_dir, exist_ok=True)
# Create a timestamp for unique backup filename
# Format: YYYYMMDD_HHMMSS
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
# Get the path of the current database from Django settings
db_path = settings.DATABASES['default']['NAME']
# Generate the backup filepath with timestamp
backup_path = os.path.join(backup_dir, f'backup_{timestamp}.db')
# Open connection to the source (current) database
source = sqlite3.connect(db_path)
# Create a new connection for the backup database
backup = sqlite3.connect(backup_path)
# Use SQLite's backup API to create the backup
# The backup() method is atomic and will maintain consistency
# even if the database is being written to during backup
with source, backup:
source.backup(backup)
logger.info(f"Database backup created successfully at {backup_path}")
self.stdout.write(
self.style.SUCCESS(f'Successfully created backup at {backup_path}')
)
except Exception as e:
logger.error(f"Database backup failed: {str(e)}")
self.stdout.write(
self.style.ERROR(f'Backup failed: {str(e)}')
)
2. Backup Management
Maintains a configurable number of recent backups
Automatically removes older backups
Includes backup rotation policies
Prevents disk space issues
apps/realtime_timer/management/commands/manage_backups.py
import os
import glob
from datetime import datetime, timedelta
from django.core.management.base import BaseCommand
from django.conf import settings
import logging
from django.core.management import call_command
logger = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Django management command to manage SQLite database backups.
This command:
1. Removes old backups exceeding the maximum count
2. Verifies the integrity of remaining backups
Usage:
python manage.py manage_backups
Configuration is read from settings.SQLITE_BACKUP:
BACKUP_DIRECTORY: Where backups are stored
BACKUP_MAX_COUNT: Maximum number of backups to keep
"""
help = 'Manages SQLite database backups by removing old ones'
def handle(self, *args, **options):
try:
# Get backup settings from Django settings
backup_dir = settings.SQLITE_BACKUP['BACKUP_DIRECTORY']
max_backups = settings.SQLITE_BACKUP['BACKUP_MAX_COUNT']
# Get all backup files and sort them by modification time
# newest files first
backup_files = glob.glob(os.path.join(backup_dir, 'backup_*.db'))
backup_files.sort(key=os.path.getmtime, reverse=True)
# If we have more backups than the maximum allowed
if len(backup_files) > max_backups:
# Remove the oldest backups (those beyond max_backups)
for old_backup in backup_files[max_backups:]:
os.remove(old_backup)
logger.info(f"Removed old backup: {old_backup}")
self.stdout.write(
self.style.SUCCESS(f'Removed old backup: {old_backup}')
)
# After cleanup, verify all remaining backups
# This calls the verify_backup command for each backup file
for backup_file in backup_files[:max_backups]:
self.stdout.write(f"Verifying backup: {backup_file}")
call_command('verify_backup', backup_file=str(backup_file))
except Exception as e:
logger.error(f"Backup management failed: {str(e)}")
self.stdout.write(
self.style.ERROR(f'Backup management failed: {str(e)}')
)
3. Backup Verification
The verification process includes:
Table structure comparison
Row count validation
Foreign key constraint checks
Test restoration to temporary database
Comprehensive integrity checks
apps/realtime_timer/management/commands/verify_backup.pyimport os import sqlite3 import tempfile from django.core.management.base import BaseCommand from django.conf import settings from django.apps import apps import logging from pathlib import Path logger = logging.getLogger(__name__) class Command(BaseCommand): """ Django management command to verify the integrity of SQLite database backups. This command: 1. Creates a temporary database 2. Restores the backup to this temporary database 3. Verifies: - Table structures match the original - Row counts match - Foreign key constraints are valid Usage: python manage.py verify_backup # verifies latest backup python manage.py verify_backup --backup-file=/path/to/backup.db # verifies specific backup """ help = 'Verifies SQLite backup integrity and tests restoration' def add_arguments(self, parser): # Allow specifying a particular backup file to verify parser.add_argument( '--backup-file', help='Specific backup file to verify. If not provided, verifies latest backup.' ) def verify_table_structure(self, source_conn, test_conn, table_name): """ Compare table schemas between original and restored databases. Uses SQLite's sqlite_master table to get the CREATE TABLE statements and compares them. """ source_schema = source_conn.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name=?", (table_name,)).fetchone() test_schema = test_conn.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name=?", (table_name,)).fetchone() return source_schema == test_schema def verify_row_counts(self, source_conn, test_conn, table_name): """ Compare the number of rows in each table between original and restored databases. A simple COUNT(*) should match if the backup is valid. """ source_count = source_conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0] test_count = test_conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0] return source_count == test_count def verify_foreign_keys(self, conn): """ Verify that all foreign key constraints are valid in the restored database. Uses SQLite's PRAGMA foreign_key_check which returns empty list if all constraints are valid. """ return conn.execute("PRAGMA foreign_key_check").fetchall() == [] def handle(self, *args, **options): try: backup_dir = settings.SQLITE_BACKUP['BACKUP_DIRECTORY'] # Determine which backup file to verify if options['backup_file']: backup_path = Path(options['backup_file']) if not backup_path.exists(): raise FileNotFoundError(f"Backup file not found: {backup_path}") else: # If no specific file provided, get the most recent backup backup_files = sorted( Path(backup_dir).glob('backup_*.db'), key=lambda x: x.stat().st_mtime, reverse=True ) if not backup_files: raise FileNotFoundError("No backup files found") backup_path = backup_files[0] # Create a temporary database file for testing restoration with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as temp_db: temp_db_path = temp_db.name try: # Connect to both backup and temporary database backup_conn = sqlite3.connect(str(backup_path)) test_conn = sqlite3.connect(temp_db_path) # Restore the backup to temporary database with backup_conn, test_conn: backup_conn.backup(test_conn) # Get all table names from Django models django_tables = [ model._meta.db_table for model in apps.get_models() ] # Store verification results verification_results = { 'structure': [], # Table structure verification results 'row_counts': [], # Row count verification results 'foreign_keys': True # Foreign key constraint verification } # Verify each table for table in django_tables: # Check if table structures match structure_match = self.verify_table_structure(backup_conn, test_conn, table) verification_results['structure'].append({ 'table': table, 'matches': structure_match }) # Check if row counts match count_match = self.verify_row_counts(backup_conn, test_conn, table) verification_results['row_counts'].append({ 'table': table, 'matches': count_match }) # Verify foreign key constraints verification_results['foreign_keys'] = self.verify_foreign_keys(test_conn) # Check if all verifications passed all_passed = ( all(r['matches'] for r in verification_results['structure']) and all(r['matches'] for r in verification_results['row_counts']) and verification_results['foreign_keys'] ) if all_passed: # All checks passed - backup is valid logger.info(f"Backup verification successful for {backup_path}") self.stdout.write( self.style.SUCCESS(f'Backup verification successful for {backup_path}') ) else: # Collect all failed checks for detailed error reporting failed_checks = [] for result in verification_results['structure']: if not result['matches']: failed_checks.append(f"Structure mismatch in table {result['table']}") for result in verification_results['row_counts']: if not result['matches']: failed_checks.append(f"Row count mismatch in table {result['table']}") if not verification_results['foreign_keys']: failed_checks.append("Foreign key constraints validation failed") logger.error(f"Backup verification failed for {backup_path}: {', '.join(failed_checks)}") self.stdout.write( self.style.ERROR(f'Backup verification failed: {", ".join(failed_checks)}') ) finally: # Clean up: remove temporary database if os.path.exists(temp_db_path): os.unlink(temp_db_path) except Exception as e: logger.error(f"Backup verification failed: {str(e)}") self.stdout.write( self.style.ERROR(f'Backup verification failed: {str(e)}') )
4. GitHub Integration
Automatic commits with readable timestamps
Configurable Git user settings
Push to dedicated backup branch
Error handling and logging
apps/realtime_timer/management/commands/backup_git_push.py
import os
import subprocess
from datetime import datetime
from django.core.management.base import BaseCommand
from django.conf import settings
import logging
logger = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Django management command to commit and push database backups to GitHub.
This command:
1. Stages all new backup files
2. Creates a commit with timestamp
3. Pushes to the configured remote branch
Usage:
python manage.py backup_git_push
Required Environment Variables:
GIT_USER_NAME: Git user name for commits
GIT_USER_EMAIL: Git user email for commits
GIT_BRANCH: Branch to push backups (default: backup)
"""
help = 'Commits and pushes database backups to GitHub'
def add_arguments(self, parser):
parser.add_argument(
'--force',
action='store_true',
help='Skip confirmation prompt and force push backup',
)
def execute_git_command(self, command, error_message):
"""Execute git command and handle errors"""
try:
result = subprocess.run(
command,
cwd=settings.BASE_DIR,
check=True,
capture_output=True,
text=True
)
return result.stdout.strip()
except subprocess.CalledProcessError as e:
logger.error(f"{error_message}: {e.stderr}")
raise
def handle(self, *args, **options):
try:
backup_dir = settings.SQLITE_BACKUP['BACKUP_DIRECTORY']
git_user = settings.GIT_USER_NAME
git_email = settings.GIT_USER_EMAIL
if not all([git_user, git_email]):
raise ValueError("GIT_USER_NAME and GIT_USER_EMAIL must be set in environment")
# Configure git user for this commit
self.execute_git_command(
['git', 'config', 'user.name', git_user],
"Failed to set git user name"
)
self.execute_git_command(
['git', 'config', 'user.email', git_email],
"Failed to set git user email"
)
# Add confirmation prompt unless --force flag is used
if not options['force']:
confirm = input(
"\nWARNING: You are about to push database backups to a remote repository.\n"
"This may override existing data in the repository.\n"
"Are you sure you want to continue? [y/N]: "
).lower()
if confirm not in ['y', 'yes']:
self.stdout.write(self.style.WARNING('Backup push cancelled'))
return
# Stage all backup files
backup_pattern = os.path.join(backup_dir, 'backup_*.db')
self.execute_git_command(
['git', 'add', backup_pattern],
"Failed to stage backup files"
)
# Create commit with timestamp
# this time is in UTC+0
timestamp = datetime.now().strftime('%B %d, %Y %I:%M %p')
commit_message = f"Database backup - {timestamp}"
self.execute_git_command(
['git', 'commit', '-m', commit_message],
"Failed to create commit"
)
# Push to remote
self.execute_git_command(
['git', 'push', 'origin'],
"Failed to push to GitHub"
)
logger.info("Successfully pushed backup to GitHub")
self.stdout.write(
self.style.SUCCESS('Successfully pushed backup to GitHub')
)
except Exception as e:
logger.error(f"Failed to push backup to GitHub: {str(e)}")
self.stdout.write(
self.style.ERROR('Failed to push backup to GitHub')
)
5. Centralized Management
A single command orchestrates:
Backup creation
Rotation management
Integrity verification
GitHub synchronization
apps/realtime_timer/management/commands/backup_manager.py
import logging
from django.core.management.base import BaseCommand
from django.core.management import call_command
from datetime import datetime
logger = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Django management command to handle all backup-related operations:
1. Creates a new backup
2. Manages backup retention (removes old backups)
3. Verifies backup integrity
4. Pushes backups to GitHub
Usage:
python manage.py backup_manager
"""
help = 'Manages all backup operations (create, verify, rotate, and push to GitHub)'
def handle(self, *args, **options):
try:
timestamp = datetime.now().strftime('%B %d, %Y %I:%M %p')
self.stdout.write(f"Starting backup operations at {timestamp}",style_func=self.style.WARNING)
# Step 1: Create new backup
self.stdout.write("Creating new backup...",style_func=self.style.WARNING)
call_command('backup_database')
# Step 2: Manage backups (includes verification)
self.stdout.write("Managing and verifying backups...",style_func=self.style.WARNING)
call_command('manage_backups')
# Step 3: Push to GitHub
self.stdout.write("Pushing backups to GitHub...",style_func=self.style.WARNING)
call_command('backup_git_push')
self.stdout.write(
self.style.SUCCESS('All backup operations completed successfully')
)
except Exception as e:
logger.error(f"Backup operations failed: {str(e)}")
self.stdout.write(
self.style.ERROR(f'Backup operations failed: {str(e)}')
)
Configuration
Settings in Django
# SQLite backup settings
SQLITE_BACKUP = {
'BACKUP_DIRECTORY': os.path.join(BASE_DIR, 'backups'),
'BACKUP_MAX_COUNT': 10, # Maximum number of backups to keep
}
GIT_USER_NAME = os.environ.get('GIT_USER_NAME')
GIT_USER_EMAIL = os.environ.get('GIT_USER_EMAIL')
GIT_USER_NAME: Git username for commitsGIT_USER_EMAIL: Git email for commits
Automation
Cron Job Setup
Run backups every 6 hours:
0 */6 * * * cd /home/normal_user/focus-timer-v/ && . /home/normal_user/focus-timer-v/.env && /home/normal_user/focus-timer-v/venv/bin/python manage.py backup_manager >> /home/normal_user/focus-timer-v/logs/crontab_logs.txt 2>&1; echo "Cron ran at $(date)" >> /home/normal_user/focus-timer-v/logs/debug_cron.log
Here is an screenshot showing it in action 👇🏻
Conclusion
This backup system provides a robust, automated solution for SQLite database backups in Django applications. It ensures data safety through multiple verification layers and maintains backup integrity while requiring minimal manual intervention.