From 617ff710e389974a26d30d53872355433e4787a7 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 23 Jul 2026 19:11:40 +0200 Subject: [PATCH 01/14] fix: backup library before making any changes --- src/tagstudio/core/library/alchemy/library.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 41ada274f..ae9c8be57 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -541,6 +541,13 @@ def open_sqlite_library( ) logger.info(f"[Library] Library DB version: {loaded_db_version}") + + # save backup if patches will be applied + if loaded_db_version < DB_VERSION: + self.library_dir = library_dir + self.save_library_backup_to_disk() + self.library_dir = None + # TODO: this is very sketchy; blindly creating all tables the newest DB version should have # without considering what version the DB is currently on and then doing all of the # migrations after that seems like it could cause problems in some scenarios. @@ -549,12 +556,6 @@ def open_sqlite_library( # a library that doesn't yet have the is_hidden property on the tags table make_tables(self.engine) - # save backup if patches will be applied - if loaded_db_version < DB_VERSION: - self.library_dir = library_dir - self.save_library_backup_to_disk() - self.library_dir = None - # migrate DB step by step from one version to the next # (migration_method, db_version, initial_db_version) migrations = [ From 0d1597d5207dd8197f24b9dab3ef4ef3d10cc6b4 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 23 Jul 2026 19:27:48 +0200 Subject: [PATCH 02/14] refactor: inline make_tables + minor cleanup --- src/tagstudio/core/library/alchemy/db.py | 40 +----------- src/tagstudio/core/library/alchemy/library.py | 63 ++++++++++++++++--- 2 files changed, 55 insertions(+), 48 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/db.py b/src/tagstudio/core/library/alchemy/db.py index d8d520d78..e28880c3d 100644 --- a/src/tagstudio/core/library/alchemy/db.py +++ b/src/tagstudio/core/library/alchemy/db.py @@ -6,12 +6,9 @@ from typing import override import structlog -from sqlalchemy import Dialect, Engine, String, TypeDecorator, create_engine, text -from sqlalchemy.exc import OperationalError +from sqlalchemy import Dialect, String, TypeDecorator from sqlalchemy.orm import DeclarativeBase -from tagstudio.core.constants import RESERVED_TAG_END - logger = structlog.getLogger(__name__) @@ -34,38 +31,3 @@ def process_result_value(self, value: str | None, dialect: Dialect): class Base(DeclarativeBase): type_annotation_map = {Path: PathType} - - -def make_engine(connection_string: str) -> Engine: - return create_engine(connection_string) - - -def make_tables(engine: Engine) -> None: - logger.info("[Library] Creating DB tables...") - with engine.connect() as conn: - # TODO: this should instead be migrations that create the exact tables that were added in - # the respective DB versions - Base.metadata.create_all(conn) - conn.commit() - - # TODO: this needs to be a migration - # tag IDs < 1000 are reserved - # create tag and delete it to bump the autoincrement sequence - # TODO - find a better way - # is this the better way? - result = conn.execute(text("SELECT SEQ FROM sqlite_sequence WHERE name='tags'")) - autoincrement_val = result.scalar() - if not autoincrement_val or autoincrement_val <= RESERVED_TAG_END: - try: - conn.execute( - text( - "INSERT INTO tags " - "(id, name, color_namespace, color_slug, is_category, is_hidden) VALUES " - f"({RESERVED_TAG_END}, 'temp', NULL, NULL, false, false)" - ) - ) - conn.execute(text(f"DELETE FROM tags WHERE id = {RESERVED_TAG_END}")) - conn.commit() - except OperationalError as e: - logger.error("Could not initialize built-in tags", error=e) - conn.rollback() diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index ae9c8be57..c3f8dc615 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -44,7 +44,7 @@ update, ) from sqlalchemy.dialects import sqlite -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.orm import ( InstanceState, Session, @@ -76,7 +76,7 @@ SQL_FILENAME, TAG_CHILDREN_QUERY, ) -from tagstudio.core.library.alchemy.db import make_tables +from tagstudio.core.library.alchemy.db import Base as ModelBase from tagstudio.core.library.alchemy.enums import ( MAX_SQL_VARIABLES, BrowsingState, @@ -431,21 +431,40 @@ def create_sqlite_library( self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME ) -> LibraryStatus: self.engine = self.__get_engine(library_dir, in_memory, sql_filename) - loaded_db_version: int = 0 logger.info( "[Library] Opening SQLite Library", library_dir=library_dir, ) - logger.info(f"[Library] Library DB version: {loaded_db_version}") - make_tables(self.engine) + logger.info("[Library] Creating DB tables...") + with self.engine.connect() as conn: + ModelBase.metadata.create_all(conn) + conn.commit() + + # TODO - find a better way + # is this the better way? + result = conn.execute(text("SELECT SEQ FROM sqlite_sequence WHERE name='tags'")) + autoincrement_val = result.scalar() + if not autoincrement_val or autoincrement_val <= RESERVED_TAG_END: + try: + conn.execute( + text( + "INSERT INTO tags " + "(id, name, color_namespace, color_slug, is_category, is_hidden) " + f"VALUES ({RESERVED_TAG_END}, 'temp', NULL, NULL, false, false)" + ) + ) + conn.execute(text(f"DELETE FROM tags WHERE id = {RESERVED_TAG_END}")) + conn.commit() + except OperationalError as e: + logger.error("Could not initialize built-in tags", error=e) + conn.rollback() with Session(self.engine) as session: # Add default tag color namespaces. namespaces = default_color_groups.namespaces() - # TODO: are all of these commits necessary? session.add_all(namespaces) session.flush() @@ -507,8 +526,6 @@ def open_sqlite_library( self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME ) -> LibraryStatus: self.engine = self.__get_engine(library_dir, in_memory, sql_filename) - loaded_db_version: int = 0 - initial_db_version: int = DB_VERSION logger.info( "[Library] Opening SQLite Library", @@ -554,7 +571,35 @@ def open_sqlite_library( # instead only have this on creation and create new tables as part of migrations # Note: this actually produces an error and fails to initialise built-in tags when opening # a library that doesn't yet have the is_hidden property on the tags table - make_tables(self.engine) + logger.info("[Library] Creating DB tables...") + with self.engine.connect() as conn: + # TODO: this should instead be migrations that create the exact tables + # that were added in the respective DB versions + ModelBase.metadata.create_all(conn) + conn.commit() + + # TODO: this needs to be a migration + # tag IDs < 1000 are reserved + # create tag and delete it to bump the autoincrement sequence + # TODO - find a better way + # is this the better way? + with self.engine.connect() as conn: + result = conn.execute(text("SELECT SEQ FROM sqlite_sequence WHERE name='tags'")) + autoincrement_val = result.scalar() + if not autoincrement_val or autoincrement_val <= RESERVED_TAG_END: + try: + conn.execute( + text( + "INSERT INTO tags " + "(id, name, color_namespace, color_slug, is_category, is_hidden) " + f"VALUES ({RESERVED_TAG_END}, 'temp', NULL, NULL, false, false)" + ) + ) + conn.execute(text(f"DELETE FROM tags WHERE id = {RESERVED_TAG_END}")) + conn.commit() + except OperationalError as e: + logger.error("Could not initialize built-in tags", error=e) + conn.rollback() # migrate DB step by step from one version to the next # (migration_method, db_version, initial_db_version) From 24f9f27e6370dfb077223859e1788237f202d07a Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 23 Jul 2026 19:37:12 +0200 Subject: [PATCH 03/14] refactor: remove unnecessary assurance Bumping the auto increment value has been done since the original sql PR, so it doesn't need to be done on migrations. See e5e7b8afc62c1a958c172b35819caa26995f2060. --- src/tagstudio/core/library/alchemy/library.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index c3f8dc615..a4667133f 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -578,29 +578,6 @@ def open_sqlite_library( ModelBase.metadata.create_all(conn) conn.commit() - # TODO: this needs to be a migration - # tag IDs < 1000 are reserved - # create tag and delete it to bump the autoincrement sequence - # TODO - find a better way - # is this the better way? - with self.engine.connect() as conn: - result = conn.execute(text("SELECT SEQ FROM sqlite_sequence WHERE name='tags'")) - autoincrement_val = result.scalar() - if not autoincrement_val or autoincrement_val <= RESERVED_TAG_END: - try: - conn.execute( - text( - "INSERT INTO tags " - "(id, name, color_namespace, color_slug, is_category, is_hidden) " - f"VALUES ({RESERVED_TAG_END}, 'temp', NULL, NULL, false, false)" - ) - ) - conn.execute(text(f"DELETE FROM tags WHERE id = {RESERVED_TAG_END}")) - conn.commit() - except OperationalError as e: - logger.error("Could not initialize built-in tags", error=e) - conn.rollback() - # migrate DB step by step from one version to the next # (migration_method, db_version, initial_db_version) migrations = [ From c5402e6bda151ba19fe4013378d806555d740922 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 23 Jul 2026 20:23:21 +0200 Subject: [PATCH 04/14] refactor: don't blindly create all tables in the beginning The only table that has been added since DB version 6 (the earliest supported version), is the versions table in DB version 101. This commit removes the "create all tables" statement, and instead creates the versions table in the 101 migration. See 12e074b71d8860282b44e49e0e1a41b7a2e4bae8. --- src/tagstudio/core/library/alchemy/library.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index a4667133f..3b6fb70b7 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -444,6 +444,7 @@ def create_sqlite_library( # TODO - find a better way # is this the better way? + # Could we perhaps update the row we are reading from here? result = conn.execute(text("SELECT SEQ FROM sqlite_sequence WHERE name='tags'")) autoincrement_val = result.scalar() if not autoincrement_val or autoincrement_val <= RESERVED_TAG_END: @@ -565,19 +566,6 @@ def open_sqlite_library( self.save_library_backup_to_disk() self.library_dir = None - # TODO: this is very sketchy; blindly creating all tables the newest DB version should have - # without considering what version the DB is currently on and then doing all of the - # migrations after that seems like it could cause problems in some scenarios. - # instead only have this on creation and create new tables as part of migrations - # Note: this actually produces an error and fails to initialise built-in tags when opening - # a library that doesn't yet have the is_hidden property on the tags table - logger.info("[Library] Creating DB tables...") - with self.engine.connect() as conn: - # TODO: this should instead be migrations that create the exact tables - # that were added in the respective DB versions - ModelBase.metadata.create_all(conn) - conn.commit() - # migrate DB step by step from one version to the next # (migration_method, db_version, initial_db_version) migrations = [ @@ -705,6 +693,16 @@ def __apply_db100_migration(self, session: Session, library_dir: Path): def __apply_db101_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 101.""" + # Create versions table + session.execute( + text(""" + CREATE TABLE versions ( + "key" VARCHAR NOT NULL PRIMARY KEY, + value INTEGER NOT NULL, + ); + """) + ) + session.flush() # Ensure version rows are present session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) session.flush() From a22482df9eca431eec23b40164cf03bc0411a418 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 23 Jul 2026 20:33:38 +0200 Subject: [PATCH 05/14] refactor: don't require setting library_dir to create a backup --- src/tagstudio/core/library/alchemy/library.py | 20 ++++++++----------- src/tagstudio/qt/ts_qt.py | 2 +- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 3b6fb70b7..caec8f954 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -528,10 +528,7 @@ def open_sqlite_library( ) -> LibraryStatus: self.engine = self.__get_engine(library_dir, in_memory, sql_filename) - logger.info( - "[Library] Opening SQLite Library", - library_dir=library_dir, - ) + logger.info("[Library] Opening SQLite Library", library_dir=library_dir) # Don't check DB version when creating new library loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) @@ -562,9 +559,7 @@ def open_sqlite_library( # save backup if patches will be applied if loaded_db_version < DB_VERSION: - self.library_dir = library_dir - self.save_library_backup_to_disk() - self.library_dir = None + Library.save_library_backup_to_disk(library_dir) # migrate DB step by step from one version to the next # (migration_method, db_version, initial_db_version) @@ -1859,16 +1854,17 @@ def delete_color(self, color: TagColorGroup): session.rollback() return None - def save_library_backup_to_disk(self) -> Path: - assert isinstance(self.library_dir, Path) - makedirs(str(self.library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME), exist_ok=True) + @staticmethod + def save_library_backup_to_disk(library_dir: Path) -> Path: + assert isinstance(library_dir, Path) + makedirs(str(library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME), exist_ok=True) filename = f"ts_library_backup_{datetime.now(UTC).strftime('%Y_%m_%d_%H%M%S')}.sqlite" - target_path = self.library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME / filename + target_path = library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME / filename shutil.copy2( - self.library_dir / TS_FOLDER_NAME / SQL_FILENAME, + library_dir / TS_FOLDER_NAME / SQL_FILENAME, target_path, ) diff --git a/src/tagstudio/qt/ts_qt.py b/src/tagstudio/qt/ts_qt.py index c0a7de1ca..7bcf75b82 100644 --- a/src/tagstudio/qt/ts_qt.py +++ b/src/tagstudio/qt/ts_qt.py @@ -837,7 +837,7 @@ def backup_library(self): logger.info("Backing Up Library...") self.main_window.status_bar.showMessage(Translations["status.library_backup_in_progress"]) start_time = time.time() - target_path = self.lib.save_library_backup_to_disk() + target_path = Library.save_library_backup_to_disk(unwrap(self.lib.library_dir)) end_time = time.time() self.main_window.status_bar.showMessage( Translations.format( From 92809cc2253a122cec3dd0983e901748cc4848d8 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 23 Jul 2026 21:10:59 +0200 Subject: [PATCH 06/14] refactor: move migrations to different file --- .../core/library/alchemy/constants.py | 17 + src/tagstudio/core/library/alchemy/library.py | 474 +---------------- .../core/library/alchemy/migrations.py | 495 ++++++++++++++++++ 3 files changed, 528 insertions(+), 458 deletions(-) create mode 100644 src/tagstudio/core/library/alchemy/migrations.py diff --git a/src/tagstudio/core/library/alchemy/constants.py b/src/tagstudio/core/library/alchemy/constants.py index 73493c9af..140b399b9 100644 --- a/src/tagstudio/core/library/alchemy/constants.py +++ b/src/tagstudio/core/library/alchemy/constants.py @@ -4,6 +4,11 @@ from sqlalchemy import text +from tagstudio.core.library.alchemy.fields import ( + DatetimeFieldTemplate, + TextFieldTemplate, +) + SQL_FILENAME: str = "ts_library.sqlite" JSON_FILENAME: str = "ts_library.json" @@ -32,3 +37,15 @@ ) SELECT tag_id FROM ChildTags; """) + + +DEFAULT_FIELD_TEMPLATES = ( + TextFieldTemplate(name="Title"), + TextFieldTemplate(name="Author"), + TextFieldTemplate(name="Artist"), + TextFieldTemplate(name="URL"), + TextFieldTemplate(name="Description", is_multiline=True), + TextFieldTemplate(name="Notes", is_multiline=True), + TextFieldTemplate(name="Comments", is_multiline=True), + DatetimeFieldTemplate(name="Date"), +) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index caec8f954..55cbbc581 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -2,11 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-only -# NOTE: This file contains necessary use of deprecated first-party code until that -# code is removed in a future version (prefs). -# pyright: reportDeprecated=false - - import re import shutil import sys @@ -19,9 +14,7 @@ from pathlib import Path from typing import TYPE_CHECKING -import sqlalchemy import structlog -import ujson from humanfriendly import format_timespan # pyright: ignore[reportUnknownVariableType] from sqlalchemy import ( URL, @@ -72,6 +65,7 @@ DB_VERSION, DB_VERSION_CURRENT_KEY, DB_VERSION_INITIAL_KEY, + DEFAULT_FIELD_TEMPLATES, JSON_FILENAME, SQL_FILENAME, TAG_CHILDREN_QUERY, @@ -92,6 +86,7 @@ TextFieldTemplate, ) from tagstudio.core.library.alchemy.joins import TagEntry, TagParent +from tagstudio.core.library.alchemy.migrations import DBMigrations, MigrationError from tagstudio.core.library.alchemy.models import ( Entry, Namespace, @@ -104,7 +99,6 @@ from tagstudio.core.library.ignore import migrate_ext_list from tagstudio.core.library.json.library import Library as JsonLibrary from tagstudio.core.utils.types import unwrap -from tagstudio.qt.translations import Translations if TYPE_CHECKING: from sqlalchemy import Select @@ -170,20 +164,6 @@ def get_default_tags() -> tuple[Tag, ...]: return archive_tag, favorite_tag, meta_tag -def get_default_field_templates() -> tuple[BaseFieldTemplate, ...]: - """Return the default field templates for a new TagStudio library.""" - title = TextFieldTemplate(name="Title") - author = TextFieldTemplate(name="Author") - artist = TextFieldTemplate(name="Artist") - url = TextFieldTemplate(name="URL") - description = TextFieldTemplate(name="Description", is_multiline=True) - notes = TextFieldTemplate(name="Notes", is_multiline=True) - comments = TextFieldTemplate(name="Comments", is_multiline=True) - date = DatetimeFieldTemplate(name="Date") - - return title, author, artist, url, description, notes, comments, date - - # The difference in the number of default JSON tags vs default tags in the current version. DEFAULT_TAG_DIFF: int = len(get_default_tags()) - len([TAG_ARCHIVED, TAG_FAVORITE]) @@ -485,7 +465,7 @@ def create_sqlite_library( session.flush() # Add default field templates - for template in get_default_field_templates(): + for template in DEFAULT_FIELD_TEMPLATES: session.add(template) session.flush() @@ -526,410 +506,25 @@ def create_sqlite_library( def open_sqlite_library( self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME ) -> LibraryStatus: - self.engine = self.__get_engine(library_dir, in_memory, sql_filename) - logger.info("[Library] Opening SQLite Library", library_dir=library_dir) - # Don't check DB version when creating new library - loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) - initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY) - - # ======================== Library Database Version Checking ======================= - # DB_VERSION 6 is the first supported SQLite DB version. - # If the DB_VERSION is >= 100, that means it's a compound major + minor version. - # - Dividing by 100 and flooring gives the major (breaking changes) version. - # - If a DB has major version higher than the current program, don't load it. - # - If only the minor version is higher, it's still allowed to load. - if loaded_db_version < 6 or ( - loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100 - ): - mismatch_text = Translations["status.library_version_mismatch"] - found_text = Translations["status.library_version_found"] - expected_text = Translations["status.library_version_expected"] - return LibraryStatus( - success=False, - message=( - f"{mismatch_text}\n" - f"{found_text} v{loaded_db_version}, " - f"{expected_text} v{DB_VERSION}" - ), - ) + self.engine = self.__get_engine(library_dir, in_memory, sql_filename) - logger.info(f"[Library] Library DB version: {loaded_db_version}") - - # save backup if patches will be applied - if loaded_db_version < DB_VERSION: - Library.save_library_backup_to_disk(library_dir) - - # migrate DB step by step from one version to the next - # (migration_method, db_version, initial_db_version) - migrations = [ - (self.__apply_db7_migration, 7, None), # changes: value_type, tags - (self.__apply_db8_migration, 8, None), # changes: tag_colors - (self.__apply_db9_migration, 9, None), # changes: entries - (self.__apply_db100_migration, 100, None), # changes: tag_parents - (self.__apply_db101_migration, 101, None), # changes: versions - (self.__apply_db102_migration, 102, None), # changes: tag_parents - (self.__apply_db103_migration, 103, None), # changes: tags - (self.__apply_db104_migration, 104, None), # changes: deletes preferences - (self.__apply_db200_migration, 200, None), # changes: field tables - (self.__apply_db201_migration, 201, 200), # changes: field tables - (self.__apply_db202_migration, 202, None), # changes: tag_parents - (self.__apply_db300_migration, 300, None), # changes: deletes folders - ] - for migration, v, iv in migrations: - if loaded_db_version < v and (iv is None or initial_db_version < iv): - logger.info(f"[Library][Migration][{v}] Starting DB Migration") - with Session(self.engine) as session: - # any error causes transaction to rollback - migration(session, library_dir) - loaded_db_version = v - self.set_version(session, DB_VERSION_CURRENT_KEY, v) - session.commit() - logger.info(f"[Library][Migration][{v}] Completed DB Migration") + try: + migrations = DBMigrations(library_dir, self.engine) - assert loaded_db_version >= DB_VERSION, ( - "Ran all migrations, but the DB is still not on the newest version" - ) - logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") + # save backup if patches will be applied + if migrations.required: + Library.save_library_backup_to_disk(library_dir) + + migrations.run() + except MigrationError as e: + return LibraryStatus(success=False, message=e.args[0]) # everything is fine, set the library path self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) - def __apply_db7_migration(self, session: Session, _library_dir: Path): - """Migrate DB from DB_VERSION 6 to 7.""" - logger.info("[Library][Migration][7] Applying patches to DB_VERSION: 6 library...") - # Repair tags that may have a disambiguation_id pointing towards a deleted tag. - # TODO: combine into single sql statement - all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() - disam_stmt = ( - update(Tag) - .where(Tag.disambiguation_id.not_in(all_tag_ids)) - .values(disambiguation_id=None) - ) - session.execute(disam_stmt) - session.flush() - - def __apply_db8_migration(self, session: Session, library_dir: Path): - """Migrate DB from DB_VERSION 7 to 8.""" - # Add the missing color_border column to the TagColorGroups table. - session.execute( - text("ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL") - ) - session.flush() - logger.info("[Library][Migration][8] Added color_border column to tag_colors table") - - # collect new default tag colors - tag_colors: list[TagColorGroup] = [ - color - for color in default_color_groups.shades() - if color.slug in ["burgundy", "dark-teal", "dark_lavender"] - ] - - # Add any new default colors introduced in DB_VERSION 8 - for color in tag_colors: - session.add(color) - session.flush() - logger.info( - "[Library][Migration][8] Migrated tag colors to DB_VERSION 8+", - color_name=tag_colors, - ) - - # Update Neon colors to use the the color_border property - for color in default_color_groups.neon(): - neon_stmt = ( - update(TagColorGroup) - .where( - and_( - TagColorGroup.namespace == color.namespace, - TagColorGroup.slug == color.slug, - ) - ) - .values( - slug=color.slug, - namespace=color.namespace, - name=color.name, - primary=color.primary, - secondary=color.secondary, - color_border=color.color_border, - ) - ) - session.execute(neon_stmt) - session.flush() - - def __apply_db9_migration(self, session: Session, library_dir: Path): - """Migrate DB from DB_VERSION 8 to 9.""" - # Apply database schema changes - add_filename_column = text( - "ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''" - ) - session.execute(add_filename_column) - session.flush() - logger.info("[Library][Migration][9] Added filename column to entries table") - - # Populate the new filename column. - for entry in self.__all_entries(session): - entry.filename = entry.path.name - session.merge(entry) - session.flush() - logger.info("[Library][Migration][9] Populated filename column in entries table") - - def __apply_db100_migration(self, session: Session, library_dir: Path): - """Migrate DB to DB_VERSION 100.""" - # Repair parent-child tag relationships that are the wrong way around. - stmt = update(TagParent).values( - parent_id=TagParent.child_id, - child_id=TagParent.parent_id, - ) - session.execute(stmt) - session.flush() - logger.info("[Library][Migration][100] Refactored TagParent table") - - def __apply_db101_migration(self, session: Session, library_dir: Path): - """Migrate DB to DB_VERSION 101.""" - # Create versions table - session.execute( - text(""" - CREATE TABLE versions ( - "key" VARCHAR NOT NULL PRIMARY KEY, - value INTEGER NOT NULL, - ); - """) - ) - session.flush() - # Ensure version rows are present - session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) - session.flush() - - def __apply_db102_migration(self, session: Session, library_dir: Path): - """Migrate DB to DB_VERSION 102.""" - # delete TagParents with a dangling parent reference - stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) - session.execute(stmt) - session.flush() - logger.info("[Library][Migration][102] Verified TagParent table data") - - def __apply_db103_migration(self, session: Session, library_dir: Path): - """Migrate DB from DB_VERSION 102 to 103.""" - # add the new hidden column for tags - session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0")) - session.flush() - logger.info("[Library][Migration][103] Added is_hidden column to tags table") - - # mark the "Archived" tag as hidden - session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) - session.flush() - logger.info("[Library][Migration][103] Updated archived tag to be hidden") - - def __apply_db104_migration(self, session: Session, library_dir: Path): - """Migrate DB from DB_VERSION 103 to 104.""" - # Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist - self.__migrate_sql_to_ts_ignore(session, library_dir) - session.execute(text("DROP TABLE preferences")) - session.flush() - - def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): - # Do not continue if existing '.ts_ignore' file is found - ts_ignore = library_dir / TS_FOLDER_NAME / IGNORE_NAME - if Path(ts_ignore).exists(): - return - - # Load legacy extension data - extensions: list[str] = ujson.loads( - unwrap( - session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'")) - ) - ) - is_exclude_list: bool = unwrap( - session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'")) - ) - - with open(ts_ignore, "w") as f: - f.write(migrate_ext_list(extensions, is_exclude_list)) - - def __apply_db200_migration(self, session: Session, library_dir: Path): - """Migrate DB to DB_VERSION 200.""" - # Drop unused 'boolean_fields' and 'value_type' tables - logger.info("[Library][Migration][200] Dropping boolean_fields and value_type tables...") - session.execute(text("DROP TABLE boolean_fields")) - session.execute(text("DROP TABLE value_type")) - - # Add 'name' column to text_fields and datetime_fields tables - logger.info("[Library][Migration][200] Adding name columns to field tables...") - stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""') - session.execute(stmt) - stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""') - session.execute(stmt) - - # Drop unnecessary 'position' columns - logger.info("[Library][Migration][200] Dropping position columns to field tables...") - session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position")) - session.execute(text("ALTER TABLE text_fields DROP COLUMN position")) - - # Add 'is_multiline' column to text_fields table - logger.info("[Library][Migration][200] Adding is_multiline column to text_fields...") - stmt = text("ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0") - session.execute(stmt) - session.flush() - - # Move values from old `type_key` columns into new `name` columns - logger.info("[Library][Migration][200] Moving values from type_key columns to name...") - session.execute(text("UPDATE text_fields SET name = type_key")) - session.execute(text("UPDATE datetime_fields SET name = type_key")) - session.flush() - - # Change `name` values to title case - logger.info("[Library][Migration][200] Normalizing TextField names...") - for text_field in session.execute(select(TextField)).scalars(): - # NOTE: The only exception to the "Title Case" conversion is the "URL" field. - text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ") - logger.info("[Library][Migration][200] Normalizing DatetimeField names...") - for datetime_field in session.execute(select(DatetimeField)).scalars(): - datetime_field.name = datetime_field.name.title().replace("_", " ") - session.flush() - - # Add correct `is_multiline` values to text_fields table - logger.info("[Library][Migration][200] Updating is_multiline for legacy TEXT_BOXes...") - text_boxes = [ - x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True - ] - update_stmt = ( - update(TextField).where(TextField.name.in_(text_boxes)).values(is_multiline=True) - ) - session.execute(update_stmt) - session.flush() - - # Repair legacy "Description" fields to use is_multiline = True - logger.info("[Library][Migration][200] Repairing legacy Description fields...") - desc_stmt = ( - update(TextField) - .where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712 - .values(is_multiline=True) - ) - session.execute(desc_stmt) - - # Repair legacy "Comments" fields to use is_multiline = True - logger.info("[Library][Migration][200] Repairing legacy Comment fields...") - comm_stmt = ( - update(TextField) - .where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712 - .values(is_multiline=True) - ) - session.execute(comm_stmt) - - # Add default field templates - logger.info("[Library][Migration][200] Adding default field templates...") - for template in get_default_field_templates(): - session.add(template) - session.flush() - - # DB indices for improved performance - session.execute( - text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") - ) - session.execute( - text("CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)") - ) - session.execute( - text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)") - ) - - def __apply_db201_migration(self, session: Session, library_dir: Path): - """Migrate DB to DB_VERSION 201.""" - create_text_fields_table = text(""" - CREATE TABLE text_fields_new ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name VARCHAR NOT NULL, - entry_id INTEGER NOT NULL, - value VARCHAR, - is_multiline BOOLEAN NOT NULL, - FOREIGN KEY(entry_id) REFERENCES entries (id) - ) - """) - create_datetime_fields_table = text(""" - CREATE TABLE datetime_fields_new ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name VARCHAR NOT NULL, - entry_id INTEGER NOT NULL, - value VARCHAR, - FOREIGN KEY(entry_id) REFERENCES entries (id) - ) - """) - - logger.info("[Library][Migration][201] Dropping type_key from text_fields table...") - session.execute(create_text_fields_table) - session.flush() - session.execute( - text(""" - INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline) - SELECT id, name, entry_id, value, is_multiline - FROM text_fields - """) - ) - session.execute(text("DROP TABLE text_fields")) - session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields")) - - logger.info("[Library][Migration][201] Dropping type_key from datetime_fields table...") - session.execute(create_datetime_fields_table) - session.flush() - session.execute( - text(""" - INSERT INTO datetime_fields_new (id, name, entry_id, value) - SELECT id, name, entry_id, value - FROM datetime_fields - """) - ) - session.execute(text("DROP TABLE datetime_fields")) - session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields")) - - session.flush() - - def __apply_db202_migration(self, session: Session, library_dir: Path): - """Migrate DB to DB_VERSION 202.""" - stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) - session.execute(stmt) - session.flush() - logger.info("[Library][Migration][202] Verified TagParent table data") - - def __apply_db300_migration(self, session: Session, library_dir: Path): - ## remove folder_id column from entries table - # create new table in the desired scheme (without folder_id column) - session.execute( - text(""" - CREATE TABLE entries_new ( - id INTEGER NOT NULL, - path VARCHAR NOT NULL, - suffix VARCHAR NOT NULL, - date_created DATETIME, - date_modified DATETIME, - date_added DATETIME, - filename TEXT NOT NULL DEFAULT '', - PRIMARY KEY (id), - UNIQUE (path) - ) - """) - ) - session.flush() - # transfer data to new table - session.execute( - text(""" - INSERT INTO entries_new (id, path, suffix, date_created, date_modified, date_added, - filename) - SELECT id, path, suffix, date_created, date_modified, date_added, filename - FROM entries - """) - ) - # delete old table - session.execute(text("DROP TABLE entries")) - # rename new table to old table - session.execute(text("ALTER TABLE entries_new RENAME TO entries")) - session.flush() - - ## drop table "folders" - session.execute(text("DROP TABLE folders")) - session.flush() - @property def field_templates(self) -> Sequence[BaseFieldTemplate]: with Session(self.engine) as session: @@ -1077,7 +672,8 @@ def entries_count(self) -> int: with Session(self.engine) as session: return unwrap(session.scalar(select(func.count(Entry.id)))) - def __all_entries(self, session: Session, with_joins: bool = False) -> Iterator[Entry]: + @staticmethod + def _all_entries(session: Session, with_joins: bool = False) -> Iterator[Entry]: """Load entries without joins.""" stmt = select(Entry) if with_joins: @@ -1106,7 +702,7 @@ def __all_entries(self, session: Session, with_joins: bool = False) -> Iterator[ def all_entries(self, with_joins: bool = False) -> Iterator[Entry]: """Load entries without joins.""" with Session(self.engine) as session: - return self.__all_entries(session, with_joins) + return Library._all_entries(session, with_joins) @property def tags(self) -> list[Tag]: @@ -2150,44 +1746,6 @@ def update_parent_tags(self, tag: Tag, parent_ids: list[int] | set[int], session ) session.add(parent_tag) - def get_version(self, key: str) -> int: - """Get a version value from the DB. - - Args: - key(str): The key for the name of the version type to set. - """ - with Session(self.engine) as session: - engine = sqlalchemy.inspect(self.engine) - try: - # "Version" table added in DB_VERSION 101 - if engine and engine.has_table("versions"): - version = session.scalar(select(Version).where(Version.key == key)) - assert version - return version.value - # NOTE: The "Preferences" table has been depreciated as of TagStudio 9.5.4 - # and is set to be removed in a future release. - else: - return int( - unwrap( - session.scalar( - text("SELECT value FROM preferences WHERE key == 'DB_VERSION'") - ) - ) - ) - except Exception: - return 0 - - def set_version(self, session: Session, key: str, value: int) -> None: - """Set a version value to the DB. - - Args: - session(Session): The SQLAlchemy DB Session to use. - key(str): The key for the name of the version type to set. - value(int): The version value to set. - """ - # Insert if key has no value yet, otherwise update the value - session.merge(Version(key=key, value=value)) - def mirror_entry_fields(self, entries: list[Entry]) -> None: """Mirror fields among multiple Entry items.""" all_fields: set[BaseField] = set() diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py new file mode 100644 index 000000000..8946f727c --- /dev/null +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -0,0 +1,495 @@ +# SPDX-FileCopyrightText: (c) TagStudio Contributors +# SPDX-License-Identifier: MIT + + +from pathlib import Path + +import sqlalchemy +import structlog +import ujson +from sqlalchemy import ( + Engine, + and_, + delete, + select, + text, + update, +) +from sqlalchemy.orm import ( + Session, +) + +from tagstudio.core.constants import ( + IGNORE_NAME, + TAG_ARCHIVED, + TS_FOLDER_NAME, +) +from tagstudio.core.library.alchemy import default_color_groups +from tagstudio.core.library.alchemy.constants import ( + DB_VERSION, + DB_VERSION_CURRENT_KEY, + DB_VERSION_INITIAL_KEY, + DEFAULT_FIELD_TEMPLATES, +) +from tagstudio.core.library.alchemy.fields import ( + LEGACY_FIELD_MAP, + DatetimeField, + TextField, +) +from tagstudio.core.library.alchemy.joins import TagParent +from tagstudio.core.library.alchemy.models import ( + Tag, + TagColorGroup, + Version, +) +from tagstudio.core.library.ignore import migrate_ext_list +from tagstudio.core.utils.types import unwrap +from tagstudio.qt.translations import Translations + +logger = structlog.get_logger(__name__) + + +class MigrationError(Exception): + pass + + +class DBMigrations: + def __init__(self, library_dir: Path, engine: Engine) -> None: + self.library_dir = library_dir + self.engine = engine + + # Don't check DB version when creating new library + self.loaded_db_version = self.__get_version(DB_VERSION_CURRENT_KEY) + self.initial_db_version = self.__get_version(DB_VERSION_INITIAL_KEY) + + # ======================== Library Database Version Checking ======================= + # DB_VERSION 6 is the first supported SQLite DB version. + # If the DB_VERSION is >= 100, that means it's a compound major + minor version. + # - Dividing by 100 and flooring gives the major (breaking changes) version. + # - If a DB has major version higher than the current program, don't load it. + # - If only the minor version is higher, it's still allowed to load. + if self.loaded_db_version < 6 or ( + self.loaded_db_version >= 100 and self.loaded_db_version // 100 > DB_VERSION // 100 + ): + mismatch_text = Translations["status.library_version_mismatch"] + found_text = Translations["status.library_version_found"] + expected_text = Translations["status.library_version_expected"] + raise MigrationError( + f"{mismatch_text}\n" + f"{found_text} v{self.loaded_db_version}, " + f"{expected_text} v{DB_VERSION}" + ) + + logger.info(f"[Library] Library DB version: {self.loaded_db_version}") + + raise NotImplementedError + + @property + def required(self) -> bool: + return self.loaded_db_version < DB_VERSION + + def run(self): + # migrate DB step by step from one version to the next + # (migration_method, db_version, initial_db_version) + migrations = [ + (self.__apply_db7_migration, 7, None), # changes: value_type, tags + (self.__apply_db8_migration, 8, None), # changes: tag_colors + (self.__apply_db9_migration, 9, None), # changes: entries + (self.__apply_db100_migration, 100, None), # changes: tag_parents + (self.__apply_db101_migration, 101, None), # changes: versions + (self.__apply_db102_migration, 102, None), # changes: tag_parents + (self.__apply_db103_migration, 103, None), # changes: tags + (self.__apply_db104_migration, 104, None), # changes: deletes preferences + (self.__apply_db200_migration, 200, None), # changes: field tables + (self.__apply_db201_migration, 201, 200), # changes: field tables + (self.__apply_db202_migration, 202, None), # changes: tag_parents + (self.__apply_db300_migration, 300, None), # changes: deletes folders + ] + for migration, v, iv in migrations: + if self.loaded_db_version < v and (iv is None or self.initial_db_version < iv): + logger.info(f"[Library][Migration][{v}] Starting DB Migration") + with Session(self.engine) as session: + # any error causes transaction to rollback + migration(session, self.library_dir) + self.loaded_db_version = v + self.__set_version(session, DB_VERSION_CURRENT_KEY, v) + session.commit() + logger.info(f"[Library][Migration][{v}] Completed DB Migration") + + assert self.loaded_db_version >= DB_VERSION, ( + "Ran all migrations, but the DB is still not on the newest version" + ) + logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") + + def __get_version(self, key: str) -> int: + """Get a version value from the DB. + + Args: + key(str): The key for the name of the version type to set. + """ + with Session(self.engine) as session: + engine = sqlalchemy.inspect(self.engine) + try: + # "Version" table added in DB_VERSION 101 + if engine and engine.has_table("versions"): + version = session.scalar(select(Version).where(Version.key == key)) + assert version + return version.value + # NOTE: The "Preferences" table has been depreciated as of TagStudio 9.5.4 + # and is set to be removed in a future release. + else: + return int( + unwrap( + session.scalar( + text("SELECT value FROM preferences WHERE key == 'DB_VERSION'") + ) + ) + ) + except Exception: + return 0 + + def __set_version(self, session: Session, key: str, value: int) -> None: + """Set a version value to the DB. + + Args: + session(Session): The SQLAlchemy DB Session to use. + key(str): The key for the name of the version type to set. + value(int): The version value to set. + """ + # Insert if key has no value yet, otherwise update the value + session.merge(Version(key=key, value=value)) + + def __apply_db7_migration(self, session: Session, _library_dir: Path): + """Migrate DB from DB_VERSION 6 to 7.""" + logger.info("[Library][Migration][7] Applying patches to DB_VERSION: 6 library...") + # Repair tags that may have a disambiguation_id pointing towards a deleted tag. + # TODO: combine into single sql statement + all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() + disam_stmt = ( + update(Tag) + .where(Tag.disambiguation_id.not_in(all_tag_ids)) + .values(disambiguation_id=None) + ) + session.execute(disam_stmt) + session.flush() + + def __apply_db8_migration(self, session: Session, library_dir: Path): + """Migrate DB from DB_VERSION 7 to 8.""" + # Add the missing color_border column to the TagColorGroups table. + session.execute( + text("ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL") + ) + session.flush() + logger.info("[Library][Migration][8] Added color_border column to tag_colors table") + + # collect new default tag colors + tag_colors: list[TagColorGroup] = [ + color + for color in default_color_groups.shades() + if color.slug in ["burgundy", "dark-teal", "dark_lavender"] + ] + + # Add any new default colors introduced in DB_VERSION 8 + for color in tag_colors: + session.add(color) + session.flush() + logger.info( + "[Library][Migration][8] Migrated tag colors to DB_VERSION 8+", + color_name=tag_colors, + ) + + # Update Neon colors to use the the color_border property + for color in default_color_groups.neon(): + neon_stmt = ( + update(TagColorGroup) + .where( + and_( + TagColorGroup.namespace == color.namespace, + TagColorGroup.slug == color.slug, + ) + ) + .values( + slug=color.slug, + namespace=color.namespace, + name=color.name, + primary=color.primary, + secondary=color.secondary, + color_border=color.color_border, + ) + ) + session.execute(neon_stmt) + session.flush() + + def __apply_db9_migration(self, session: Session, library_dir: Path): + """Migrate DB from DB_VERSION 8 to 9.""" + # Apply database schema changes + add_filename_column = text( + "ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''" + ) + session.execute(add_filename_column) + session.flush() + logger.info("[Library][Migration][9] Added filename column to entries table") + + # Populate the new filename column. + from tagstudio.core.library.alchemy.library import Library + + for entry in Library._all_entries(session): + entry.filename = entry.path.name + session.merge(entry) + session.flush() + logger.info("[Library][Migration][9] Populated filename column in entries table") + + def __apply_db100_migration(self, session: Session, library_dir: Path): + """Migrate DB to DB_VERSION 100.""" + # Repair parent-child tag relationships that are the wrong way around. + stmt = update(TagParent).values( + parent_id=TagParent.child_id, + child_id=TagParent.parent_id, + ) + session.execute(stmt) + session.flush() + logger.info("[Library][Migration][100] Refactored TagParent table") + + def __apply_db101_migration(self, session: Session, library_dir: Path): + """Migrate DB to DB_VERSION 101.""" + # Create versions table + session.execute( + text(""" + CREATE TABLE versions ( + "key" VARCHAR NOT NULL PRIMARY KEY, + value INTEGER NOT NULL, + ); + """) + ) + session.flush() + # Ensure version rows are present + session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) + session.flush() + + def __apply_db102_migration(self, session: Session, library_dir: Path): + """Migrate DB to DB_VERSION 102.""" + # delete TagParents with a dangling parent reference + stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) + session.execute(stmt) + session.flush() + logger.info("[Library][Migration][102] Verified TagParent table data") + + def __apply_db103_migration(self, session: Session, library_dir: Path): + """Migrate DB from DB_VERSION 102 to 103.""" + # add the new hidden column for tags + session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0")) + session.flush() + logger.info("[Library][Migration][103] Added is_hidden column to tags table") + + # mark the "Archived" tag as hidden + session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) + session.flush() + logger.info("[Library][Migration][103] Updated archived tag to be hidden") + + def __apply_db104_migration(self, session: Session, library_dir: Path): + """Migrate DB from DB_VERSION 103 to 104.""" + # Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist + self.__migrate_sql_to_ts_ignore(session, library_dir) + session.execute(text("DROP TABLE preferences")) + session.flush() + + def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): + # Do not continue if existing '.ts_ignore' file is found + ts_ignore = library_dir / TS_FOLDER_NAME / IGNORE_NAME + if Path(ts_ignore).exists(): + return + + # Load legacy extension data + extensions: list[str] = ujson.loads( + unwrap( + session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'")) + ) + ) + is_exclude_list: bool = unwrap( + session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'")) + ) + + with open(ts_ignore, "w") as f: + f.write(migrate_ext_list(extensions, is_exclude_list)) + + def __apply_db200_migration(self, session: Session, library_dir: Path): + """Migrate DB to DB_VERSION 200.""" + # Drop unused 'boolean_fields' and 'value_type' tables + logger.info("[Library][Migration][200] Dropping boolean_fields and value_type tables...") + session.execute(text("DROP TABLE boolean_fields")) + session.execute(text("DROP TABLE value_type")) + + # Add 'name' column to text_fields and datetime_fields tables + logger.info("[Library][Migration][200] Adding name columns to field tables...") + stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""') + session.execute(stmt) + stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""') + session.execute(stmt) + + # Drop unnecessary 'position' columns + logger.info("[Library][Migration][200] Dropping position columns to field tables...") + session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position")) + session.execute(text("ALTER TABLE text_fields DROP COLUMN position")) + + # Add 'is_multiline' column to text_fields table + logger.info("[Library][Migration][200] Adding is_multiline column to text_fields...") + stmt = text("ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0") + session.execute(stmt) + session.flush() + + # Move values from old `type_key` columns into new `name` columns + logger.info("[Library][Migration][200] Moving values from type_key columns to name...") + session.execute(text("UPDATE text_fields SET name = type_key")) + session.execute(text("UPDATE datetime_fields SET name = type_key")) + session.flush() + + # Change `name` values to title case + logger.info("[Library][Migration][200] Normalizing TextField names...") + for text_field in session.execute(select(TextField)).scalars(): + # NOTE: The only exception to the "Title Case" conversion is the "URL" field. + text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ") + logger.info("[Library][Migration][200] Normalizing DatetimeField names...") + for datetime_field in session.execute(select(DatetimeField)).scalars(): + datetime_field.name = datetime_field.name.title().replace("_", " ") + session.flush() + + # Add correct `is_multiline` values to text_fields table + logger.info("[Library][Migration][200] Updating is_multiline for legacy TEXT_BOXes...") + text_boxes = [ + x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True + ] + update_stmt = ( + update(TextField).where(TextField.name.in_(text_boxes)).values(is_multiline=True) + ) + session.execute(update_stmt) + session.flush() + + # Repair legacy "Description" fields to use is_multiline = True + logger.info("[Library][Migration][200] Repairing legacy Description fields...") + desc_stmt = ( + update(TextField) + .where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712 + .values(is_multiline=True) + ) + session.execute(desc_stmt) + + # Repair legacy "Comments" fields to use is_multiline = True + logger.info("[Library][Migration][200] Repairing legacy Comment fields...") + comm_stmt = ( + update(TextField) + .where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712 + .values(is_multiline=True) + ) + session.execute(comm_stmt) + + # Add default field templates + logger.info("[Library][Migration][200] Adding default field templates...") + for template in DEFAULT_FIELD_TEMPLATES: + session.add(template) + session.flush() + + # DB indices for improved performance + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") + ) + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)") + ) + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)") + ) + + def __apply_db201_migration(self, session: Session, library_dir: Path): + """Migrate DB to DB_VERSION 201.""" + create_text_fields_table = text(""" + CREATE TABLE text_fields_new ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL, + entry_id INTEGER NOT NULL, + value VARCHAR, + is_multiline BOOLEAN NOT NULL, + FOREIGN KEY(entry_id) REFERENCES entries (id) + ) + """) + create_datetime_fields_table = text(""" + CREATE TABLE datetime_fields_new ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL, + entry_id INTEGER NOT NULL, + value VARCHAR, + FOREIGN KEY(entry_id) REFERENCES entries (id) + ) + """) + + logger.info("[Library][Migration][201] Dropping type_key from text_fields table...") + session.execute(create_text_fields_table) + session.flush() + session.execute( + text(""" + INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline) + SELECT id, name, entry_id, value, is_multiline + FROM text_fields + """) + ) + session.execute(text("DROP TABLE text_fields")) + session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields")) + + logger.info("[Library][Migration][201] Dropping type_key from datetime_fields table...") + session.execute(create_datetime_fields_table) + session.flush() + session.execute( + text(""" + INSERT INTO datetime_fields_new (id, name, entry_id, value) + SELECT id, name, entry_id, value + FROM datetime_fields + """) + ) + session.execute(text("DROP TABLE datetime_fields")) + session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields")) + + session.flush() + + def __apply_db202_migration(self, session: Session, library_dir: Path): + """Migrate DB to DB_VERSION 202.""" + stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) + session.execute(stmt) + session.flush() + logger.info("[Library][Migration][202] Verified TagParent table data") + + def __apply_db300_migration(self, session: Session, library_dir: Path): + ## remove folder_id column from entries table + # create new table in the desired scheme (without folder_id column) + session.execute( + text(""" + CREATE TABLE entries_new ( + id INTEGER NOT NULL, + path VARCHAR NOT NULL, + suffix VARCHAR NOT NULL, + date_created DATETIME, + date_modified DATETIME, + date_added DATETIME, + filename TEXT NOT NULL DEFAULT '', + PRIMARY KEY (id), + UNIQUE (path) + ) + """) + ) + session.flush() + # transfer data to new table + session.execute( + text(""" + INSERT INTO entries_new (id, path, suffix, date_created, date_modified, date_added, + filename) + SELECT id, path, suffix, date_created, date_modified, date_added, filename + FROM entries + """) + ) + # delete old table + session.execute(text("DROP TABLE entries")) + # rename new table to old table + session.execute(text("ALTER TABLE entries_new RENAME TO entries")) + session.flush() + + ## drop table "folders" + session.execute(text("DROP TABLE folders")) + session.flush() From eef0da69015140f3ee3648e4fbe95de28f897c1c Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 23 Jul 2026 21:52:26 +0200 Subject: [PATCH 07/14] refactor: package each migration in a class --- .../core/library/alchemy/migrations.py | 204 ++++++++++++------ 1 file changed, 143 insertions(+), 61 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 8946f727c..0f0405a92 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: MIT +from abc import abstractmethod +from collections.abc import Callable from pathlib import Path import sqlalchemy @@ -53,6 +55,16 @@ class MigrationError(Exception): pass +class DBMigration: + version: int = None # pyright: ignore[reportAssignmentType] + initial_version: int | None = None + + @abstractmethod + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log: Callable[[str], str]): + raise NotImplementedError + + class DBMigrations: def __init__(self, library_dir: Path, engine: Engine) -> None: self.library_dir = library_dir @@ -80,9 +92,9 @@ def __init__(self, library_dir: Path, engine: Engine) -> None: f"{expected_text} v{DB_VERSION}" ) - logger.info(f"[Library] Library DB version: {self.loaded_db_version}") - - raise NotImplementedError + logger.info( + f"[Library][Migration] Starting with library DB version: {self.loaded_db_version}" + ) @property def required(self) -> bool: @@ -91,35 +103,42 @@ def required(self) -> bool: def run(self): # migrate DB step by step from one version to the next # (migration_method, db_version, initial_db_version) - migrations = [ - (self.__apply_db7_migration, 7, None), # changes: value_type, tags - (self.__apply_db8_migration, 8, None), # changes: tag_colors - (self.__apply_db9_migration, 9, None), # changes: entries - (self.__apply_db100_migration, 100, None), # changes: tag_parents - (self.__apply_db101_migration, 101, None), # changes: versions - (self.__apply_db102_migration, 102, None), # changes: tag_parents - (self.__apply_db103_migration, 103, None), # changes: tags - (self.__apply_db104_migration, 104, None), # changes: deletes preferences - (self.__apply_db200_migration, 200, None), # changes: field tables - (self.__apply_db201_migration, 201, 200), # changes: field tables - (self.__apply_db202_migration, 202, None), # changes: tag_parents - (self.__apply_db300_migration, 300, None), # changes: deletes folders + migrations: list[type[DBMigration]] = [ + MigrationTo7, # changes: value_type, tags + MigrationTo8, # changes: tag_colors + MigrationTo9, # changes: entries + MigrationTo100, # changes: tag_parents + MigrationTo101, # changes: versions + MigrationTo102, # changes: tag_parents + MigrationTo103, # changes: tags + MigrationTo104, # changes: deletes preferences + MigrationTo200, # changes: field tables + MigrationTo201, # changes: field tables + MigrationTo202, # changes: tag_parents + MigrationTo300, # changes: deletes folders ] - for migration, v, iv in migrations: - if self.loaded_db_version < v and (iv is None or self.initial_db_version < iv): - logger.info(f"[Library][Migration][{v}] Starting DB Migration") + for migration in migrations: + if self.loaded_db_version < migration.version and ( + migration.initial_version is None + or self.initial_db_version < migration.initial_version + ): + logger.info(f"[Library][Migration][{migration.version}] Starting DB Migration") with Session(self.engine) as session: # any error causes transaction to rollback - migration(session, self.library_dir) - self.loaded_db_version = v - self.__set_version(session, DB_VERSION_CURRENT_KEY, v) + migration.run( + session, + self.library_dir, + lambda msg, v=migration.version: f"[Library][Migration][{v}] {msg}", + ) + self.loaded_db_version = migration.version + self.__set_version(session, DB_VERSION_CURRENT_KEY, migration.version) session.commit() - logger.info(f"[Library][Migration][{v}] Completed DB Migration") + logger.info(f"[Library][Migration][{migration.version}] Completed DB Migration") assert self.loaded_db_version >= DB_VERSION, ( "Ran all migrations, but the DB is still not on the newest version" ) - logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") + logger.info(f"[Library][Migration] Library migrated to DB version {DB_VERSION}") def __get_version(self, key: str) -> int: """Get a version value from the DB. @@ -159,9 +178,14 @@ def __set_version(self, session: Session, key: str, value: int) -> None: # Insert if key has no value yet, otherwise update the value session.merge(Version(key=key, value=value)) - def __apply_db7_migration(self, session: Session, _library_dir: Path): + +class MigrationTo7(DBMigration): + version = 7 + + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 6 to 7.""" - logger.info("[Library][Migration][7] Applying patches to DB_VERSION: 6 library...") + logger.info(fmt_log("Applying patches to DB_VERSION: 6 library...")) # Repair tags that may have a disambiguation_id pointing towards a deleted tag. # TODO: combine into single sql statement all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() @@ -173,14 +197,19 @@ def __apply_db7_migration(self, session: Session, _library_dir: Path): session.execute(disam_stmt) session.flush() - def __apply_db8_migration(self, session: Session, library_dir: Path): + +class MigrationTo8(DBMigration): + version = 8 + + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 7 to 8.""" # Add the missing color_border column to the TagColorGroups table. session.execute( text("ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL") ) session.flush() - logger.info("[Library][Migration][8] Added color_border column to tag_colors table") + logger.info(fmt_log("Added color_border column to tag_colors table")) # collect new default tag colors tag_colors: list[TagColorGroup] = [ @@ -194,7 +223,7 @@ def __apply_db8_migration(self, session: Session, library_dir: Path): session.add(color) session.flush() logger.info( - "[Library][Migration][8] Migrated tag colors to DB_VERSION 8+", + fmt_log("Migrated tag colors to DB_VERSION 8+"), color_name=tag_colors, ) @@ -220,7 +249,12 @@ def __apply_db8_migration(self, session: Session, library_dir: Path): session.execute(neon_stmt) session.flush() - def __apply_db9_migration(self, session: Session, library_dir: Path): + +class MigrationTo9(DBMigration): + version = 9 + + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 8 to 9.""" # Apply database schema changes add_filename_column = text( @@ -228,7 +262,7 @@ def __apply_db9_migration(self, session: Session, library_dir: Path): ) session.execute(add_filename_column) session.flush() - logger.info("[Library][Migration][9] Added filename column to entries table") + logger.info(fmt_log("Added filename column to entries table")) # Populate the new filename column. from tagstudio.core.library.alchemy.library import Library @@ -237,9 +271,14 @@ def __apply_db9_migration(self, session: Session, library_dir: Path): entry.filename = entry.path.name session.merge(entry) session.flush() - logger.info("[Library][Migration][9] Populated filename column in entries table") + logger.info(fmt_log("Populated filename column in entries table")) + + +class MigrationTo100(DBMigration): + version = 100 - def __apply_db100_migration(self, session: Session, library_dir: Path): + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 100.""" # Repair parent-child tag relationships that are the wrong way around. stmt = update(TagParent).values( @@ -248,9 +287,14 @@ def __apply_db100_migration(self, session: Session, library_dir: Path): ) session.execute(stmt) session.flush() - logger.info("[Library][Migration][100] Refactored TagParent table") + logger.info(fmt_log("Refactored TagParent table")) - def __apply_db101_migration(self, session: Session, library_dir: Path): + +class MigrationTo101(DBMigration): + version = 101 + + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 101.""" # Create versions table session.execute( @@ -265,35 +309,52 @@ def __apply_db101_migration(self, session: Session, library_dir: Path): # Ensure version rows are present session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) session.flush() + logger.info(fmt_log("Created versions table")) + + +class MigrationTo102(DBMigration): + version = 102 - def __apply_db102_migration(self, session: Session, library_dir: Path): + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 102.""" # delete TagParents with a dangling parent reference stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) session.execute(stmt) session.flush() - logger.info("[Library][Migration][102] Verified TagParent table data") + logger.info(fmt_log("Verified TagParent table data")) - def __apply_db103_migration(self, session: Session, library_dir: Path): + +class MigrationTo103(DBMigration): + version = 103 + + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 102 to 103.""" # add the new hidden column for tags session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0")) session.flush() - logger.info("[Library][Migration][103] Added is_hidden column to tags table") + logger.info(fmt_log("Added is_hidden column to tags table")) # mark the "Archived" tag as hidden session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) session.flush() - logger.info("[Library][Migration][103] Updated archived tag to be hidden") + logger.info(fmt_log("Updated archived tag to be hidden")) + + +class MigrationTo104(DBMigration): + version = 104 - def __apply_db104_migration(self, session: Session, library_dir: Path): + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 103 to 104.""" # Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist - self.__migrate_sql_to_ts_ignore(session, library_dir) + cls.__migrate_sql_to_ts_ignore(session, library_dir) session.execute(text("DROP TABLE preferences")) session.flush() - def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): + @classmethod + def __migrate_sql_to_ts_ignore(cls, session: Session, library_dir: Path): # Do not continue if existing '.ts_ignore' file is found ts_ignore = library_dir / TS_FOLDER_NAME / IGNORE_NAME if Path(ts_ignore).exists(): @@ -312,49 +373,54 @@ def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): with open(ts_ignore, "w") as f: f.write(migrate_ext_list(extensions, is_exclude_list)) - def __apply_db200_migration(self, session: Session, library_dir: Path): + +class MigrationTo200(DBMigration): + version = 200 + + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 200.""" # Drop unused 'boolean_fields' and 'value_type' tables - logger.info("[Library][Migration][200] Dropping boolean_fields and value_type tables...") + logger.info(fmt_log("Dropping boolean_fields and value_type tables...")) session.execute(text("DROP TABLE boolean_fields")) session.execute(text("DROP TABLE value_type")) # Add 'name' column to text_fields and datetime_fields tables - logger.info("[Library][Migration][200] Adding name columns to field tables...") + logger.info(fmt_log("Adding name columns to field tables...")) stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""') session.execute(stmt) stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""') session.execute(stmt) # Drop unnecessary 'position' columns - logger.info("[Library][Migration][200] Dropping position columns to field tables...") + logger.info(fmt_log("Dropping position columns to field tables...")) session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position")) session.execute(text("ALTER TABLE text_fields DROP COLUMN position")) # Add 'is_multiline' column to text_fields table - logger.info("[Library][Migration][200] Adding is_multiline column to text_fields...") + logger.info(fmt_log("Adding is_multiline column to text_fields...")) stmt = text("ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0") session.execute(stmt) session.flush() # Move values from old `type_key` columns into new `name` columns - logger.info("[Library][Migration][200] Moving values from type_key columns to name...") + logger.info(fmt_log("Moving values from type_key columns to name...")) session.execute(text("UPDATE text_fields SET name = type_key")) session.execute(text("UPDATE datetime_fields SET name = type_key")) session.flush() # Change `name` values to title case - logger.info("[Library][Migration][200] Normalizing TextField names...") + logger.info(fmt_log("Normalizing TextField names...")) for text_field in session.execute(select(TextField)).scalars(): # NOTE: The only exception to the "Title Case" conversion is the "URL" field. text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ") - logger.info("[Library][Migration][200] Normalizing DatetimeField names...") + logger.info(fmt_log("Normalizing DatetimeField names...")) for datetime_field in session.execute(select(DatetimeField)).scalars(): datetime_field.name = datetime_field.name.title().replace("_", " ") session.flush() # Add correct `is_multiline` values to text_fields table - logger.info("[Library][Migration][200] Updating is_multiline for legacy TEXT_BOXes...") + logger.info(fmt_log("Updating is_multiline for legacy TEXT_BOXes...")) text_boxes = [ x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True ] @@ -365,7 +431,7 @@ def __apply_db200_migration(self, session: Session, library_dir: Path): session.flush() # Repair legacy "Description" fields to use is_multiline = True - logger.info("[Library][Migration][200] Repairing legacy Description fields...") + logger.info(fmt_log("Repairing legacy Description fields...")) desc_stmt = ( update(TextField) .where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712 @@ -374,7 +440,7 @@ def __apply_db200_migration(self, session: Session, library_dir: Path): session.execute(desc_stmt) # Repair legacy "Comments" fields to use is_multiline = True - logger.info("[Library][Migration][200] Repairing legacy Comment fields...") + logger.info(fmt_log("Repairing legacy Comment fields...")) comm_stmt = ( update(TextField) .where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712 @@ -383,7 +449,7 @@ def __apply_db200_migration(self, session: Session, library_dir: Path): session.execute(comm_stmt) # Add default field templates - logger.info("[Library][Migration][200] Adding default field templates...") + logger.info(fmt_log("Adding default field templates...")) for template in DEFAULT_FIELD_TEMPLATES: session.add(template) session.flush() @@ -399,7 +465,13 @@ def __apply_db200_migration(self, session: Session, library_dir: Path): text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)") ) - def __apply_db201_migration(self, session: Session, library_dir: Path): + +class MigrationTo201(DBMigration): + version = 201 + initial_version = 200 + + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 201.""" create_text_fields_table = text(""" CREATE TABLE text_fields_new ( @@ -421,7 +493,7 @@ def __apply_db201_migration(self, session: Session, library_dir: Path): ) """) - logger.info("[Library][Migration][201] Dropping type_key from text_fields table...") + logger.info(fmt_log("Dropping type_key from text_fields table...")) session.execute(create_text_fields_table) session.flush() session.execute( @@ -434,7 +506,7 @@ def __apply_db201_migration(self, session: Session, library_dir: Path): session.execute(text("DROP TABLE text_fields")) session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields")) - logger.info("[Library][Migration][201] Dropping type_key from datetime_fields table...") + logger.info(fmt_log("Dropping type_key from datetime_fields table...")) session.execute(create_datetime_fields_table) session.flush() session.execute( @@ -449,14 +521,24 @@ def __apply_db201_migration(self, session: Session, library_dir: Path): session.flush() - def __apply_db202_migration(self, session: Session, library_dir: Path): + +class MigrationTo202(DBMigration): + version = 202 + + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 202.""" stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) session.execute(stmt) session.flush() - logger.info("[Library][Migration][202] Verified TagParent table data") + logger.info(fmt_log("Verified TagParent table data")) + + +class MigrationTo300(DBMigration): + version = 300 - def __apply_db300_migration(self, session: Session, library_dir: Path): + @classmethod + def run(cls, session: Session, library_dir: Path, fmt_log): ## remove folder_id column from entries table # create new table in the desired scheme (without folder_id column) session.execute( From 75c586db79a5f714225b09159d87dcf71ec7e8d2 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 23 Jul 2026 22:09:38 +0200 Subject: [PATCH 08/14] fix: some syntax errors had slipped through --- src/tagstudio/core/library/alchemy/library.py | 44 ++++++++++++++++ .../core/library/alchemy/migrations.py | 51 +++---------------- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 55cbbc581..2d1c0d884 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -14,6 +14,7 @@ from pathlib import Path from typing import TYPE_CHECKING +import sqlalchemy import structlog from humanfriendly import format_timespan # pyright: ignore[reportUnknownVariableType] from sqlalchemy import ( @@ -1746,6 +1747,49 @@ def update_parent_tags(self, tag: Tag, parent_ids: list[int] | set[int], session ) session.add(parent_tag) + def get_version(self, key: str) -> int: + """Get a version value from the DB. + + Args: + key(str): The key for the name of the version type to set. + """ + return Library._get_version(self.engine, key) + + @staticmethod + def _get_version(engine, key: str) -> int: + with Session(engine) as session: + engine = sqlalchemy.inspect(engine) + try: + # "Version" table added in DB_VERSION 101 + if engine and engine.has_table("versions"): + version = session.scalar(select(Version).where(Version.key == key)) + assert version + return version.value + # NOTE: The "Preferences" table has been depreciated as of TagStudio 9.5.4 + # and is set to be removed in a future release. + else: + return int( + unwrap( + session.scalar( + text("SELECT value FROM preferences WHERE key == 'DB_VERSION'") + ) + ) + ) + except Exception: + return 0 + + @staticmethod + def _set_version(session: Session, key: str, value: int) -> None: + """Set a version value to the DB. + + Args: + session(Session): The SQLAlchemy DB Session to use. + key(str): The key for the name of the version type to set. + value(int): The version value to set. + """ + # Insert if key has no value yet, otherwise update the value + session.merge(Version(key=key, value=value)) + def mirror_entry_fields(self, entries: list[Entry]) -> None: """Mirror fields among multiple Entry items.""" all_fields: set[BaseField] = set() diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 0f0405a92..27bfe43c3 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -2,11 +2,9 @@ # SPDX-License-Identifier: MIT -from abc import abstractmethod from collections.abc import Callable from pathlib import Path -import sqlalchemy import structlog import ujson from sqlalchemy import ( @@ -59,7 +57,6 @@ class DBMigration: version: int = None # pyright: ignore[reportAssignmentType] initial_version: int | None = None - @abstractmethod @classmethod def run(cls, session: Session, library_dir: Path, fmt_log: Callable[[str], str]): raise NotImplementedError @@ -67,12 +64,14 @@ def run(cls, session: Session, library_dir: Path, fmt_log: Callable[[str], str]) class DBMigrations: def __init__(self, library_dir: Path, engine: Engine) -> None: + from tagstudio.core.library.alchemy.library import Library + self.library_dir = library_dir self.engine = engine # Don't check DB version when creating new library - self.loaded_db_version = self.__get_version(DB_VERSION_CURRENT_KEY) - self.initial_db_version = self.__get_version(DB_VERSION_INITIAL_KEY) + self.loaded_db_version = Library._get_version(engine, DB_VERSION_CURRENT_KEY) + self.initial_db_version = Library._get_version(engine, DB_VERSION_INITIAL_KEY) # ======================== Library Database Version Checking ======================= # DB_VERSION 6 is the first supported SQLite DB version. @@ -101,6 +100,8 @@ def required(self) -> bool: return self.loaded_db_version < DB_VERSION def run(self): + from tagstudio.core.library.alchemy.library import Library + # migrate DB step by step from one version to the next # (migration_method, db_version, initial_db_version) migrations: list[type[DBMigration]] = [ @@ -131,7 +132,7 @@ def run(self): lambda msg, v=migration.version: f"[Library][Migration][{v}] {msg}", ) self.loaded_db_version = migration.version - self.__set_version(session, DB_VERSION_CURRENT_KEY, migration.version) + Library._set_version(session, DB_VERSION_CURRENT_KEY, migration.version) session.commit() logger.info(f"[Library][Migration][{migration.version}] Completed DB Migration") @@ -140,44 +141,6 @@ def run(self): ) logger.info(f"[Library][Migration] Library migrated to DB version {DB_VERSION}") - def __get_version(self, key: str) -> int: - """Get a version value from the DB. - - Args: - key(str): The key for the name of the version type to set. - """ - with Session(self.engine) as session: - engine = sqlalchemy.inspect(self.engine) - try: - # "Version" table added in DB_VERSION 101 - if engine and engine.has_table("versions"): - version = session.scalar(select(Version).where(Version.key == key)) - assert version - return version.value - # NOTE: The "Preferences" table has been depreciated as of TagStudio 9.5.4 - # and is set to be removed in a future release. - else: - return int( - unwrap( - session.scalar( - text("SELECT value FROM preferences WHERE key == 'DB_VERSION'") - ) - ) - ) - except Exception: - return 0 - - def __set_version(self, session: Session, key: str, value: int) -> None: - """Set a version value to the DB. - - Args: - session(Session): The SQLAlchemy DB Session to use. - key(str): The key for the name of the version type to set. - value(int): The version value to set. - """ - # Insert if key has no value yet, otherwise update the value - session.merge(Version(key=key, value=value)) - class MigrationTo7(DBMigration): version = 7 From 200a285de6079240ac7ad1ddfa6054305a308c8e Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 23 Jul 2026 22:24:19 +0200 Subject: [PATCH 09/14] fix: allow set_version to fail, but don't commit in that case --- .../core/library/alchemy/migrations.py | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 27bfe43c3..d484538ac 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -118,13 +118,13 @@ def run(self): MigrationTo202, # changes: tag_parents MigrationTo300, # changes: deletes folders ] - for migration in migrations: - if self.loaded_db_version < migration.version and ( - migration.initial_version is None - or self.initial_db_version < migration.initial_version - ): - logger.info(f"[Library][Migration][{migration.version}] Starting DB Migration") - with Session(self.engine) as session: + with Session(self.engine) as session: + for migration in migrations: + if self.loaded_db_version < migration.version and ( + migration.initial_version is None + or self.initial_db_version < migration.initial_version + ): + logger.info(f"[Library][Migration][{migration.version}] Starting DB Migration") # any error causes transaction to rollback migration.run( session, @@ -132,8 +132,17 @@ def run(self): lambda msg, v=migration.version: f"[Library][Migration][{v}] {msg}", ) self.loaded_db_version = migration.version - Library._set_version(session, DB_VERSION_CURRENT_KEY, migration.version) - session.commit() + try: + Library._set_version(session, DB_VERSION_CURRENT_KEY, migration.version) + except Exception as e: + logger.info( + f"[Library][Migration][{migration.version}] " + "Couldn't update version, continuing without commit", + error=e, + ) + session.flush() + else: + session.commit() logger.info(f"[Library][Migration][{migration.version}] Completed DB Migration") assert self.loaded_db_version >= DB_VERSION, ( @@ -264,8 +273,8 @@ def run(cls, session: Session, library_dir: Path, fmt_log): text(""" CREATE TABLE versions ( "key" VARCHAR NOT NULL PRIMARY KEY, - value INTEGER NOT NULL, - ); + value INTEGER NOT NULL + ) """) ) session.flush() From c7994500727570285d26c29a1e33b860f8f46ed1 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Fri, 24 Jul 2026 16:00:31 +0200 Subject: [PATCH 10/14] refactor: condense imports --- src/tagstudio/core/library/alchemy/library.py | 6 +--- .../core/library/alchemy/migrations.py | 31 +++---------------- 2 files changed, 6 insertions(+), 31 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 2d1c0d884..49cf80148 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -72,11 +72,7 @@ TAG_CHILDREN_QUERY, ) from tagstudio.core.library.alchemy.db import Base as ModelBase -from tagstudio.core.library.alchemy.enums import ( - MAX_SQL_VARIABLES, - BrowsingState, - SortingModeEnum, -) +from tagstudio.core.library.alchemy.enums import MAX_SQL_VARIABLES, BrowsingState, SortingModeEnum from tagstudio.core.library.alchemy.fields import ( LEGACY_FIELD_MAP, BaseField, diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index d484538ac..2147f29e5 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -7,23 +7,10 @@ import structlog import ujson -from sqlalchemy import ( - Engine, - and_, - delete, - select, - text, - update, -) -from sqlalchemy.orm import ( - Session, -) +from sqlalchemy import Engine, and_, delete, select, text, update +from sqlalchemy.orm import Session -from tagstudio.core.constants import ( - IGNORE_NAME, - TAG_ARCHIVED, - TS_FOLDER_NAME, -) +from tagstudio.core.constants import IGNORE_NAME, TAG_ARCHIVED, TS_FOLDER_NAME from tagstudio.core.library.alchemy import default_color_groups from tagstudio.core.library.alchemy.constants import ( DB_VERSION, @@ -31,17 +18,9 @@ DB_VERSION_INITIAL_KEY, DEFAULT_FIELD_TEMPLATES, ) -from tagstudio.core.library.alchemy.fields import ( - LEGACY_FIELD_MAP, - DatetimeField, - TextField, -) +from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP, DatetimeField, TextField from tagstudio.core.library.alchemy.joins import TagParent -from tagstudio.core.library.alchemy.models import ( - Tag, - TagColorGroup, - Version, -) +from tagstudio.core.library.alchemy.models import Tag, TagColorGroup, Version from tagstudio.core.library.ignore import migrate_ext_list from tagstudio.core.utils.types import unwrap from tagstudio.qt.translations import Translations From ccd07ee0ff585af0135c59cf5e25a9e8aff6dfab Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Fri, 24 Jul 2026 16:02:28 +0200 Subject: [PATCH 11/14] fix: add override decorators --- src/tagstudio/core/library/alchemy/migrations.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 2147f29e5..6685556d3 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -4,6 +4,7 @@ from collections.abc import Callable from pathlib import Path +from typing import override import structlog import ujson @@ -133,6 +134,7 @@ def run(self): class MigrationTo7(DBMigration): version = 7 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 6 to 7.""" @@ -152,6 +154,7 @@ def run(cls, session: Session, library_dir: Path, fmt_log): class MigrationTo8(DBMigration): version = 8 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 7 to 8.""" @@ -204,6 +207,7 @@ def run(cls, session: Session, library_dir: Path, fmt_log): class MigrationTo9(DBMigration): version = 9 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 8 to 9.""" @@ -228,6 +232,7 @@ def run(cls, session: Session, library_dir: Path, fmt_log): class MigrationTo100(DBMigration): version = 100 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 100.""" @@ -244,6 +249,7 @@ def run(cls, session: Session, library_dir: Path, fmt_log): class MigrationTo101(DBMigration): version = 101 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 101.""" @@ -266,6 +272,7 @@ def run(cls, session: Session, library_dir: Path, fmt_log): class MigrationTo102(DBMigration): version = 102 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 102.""" @@ -279,6 +286,7 @@ def run(cls, session: Session, library_dir: Path, fmt_log): class MigrationTo103(DBMigration): version = 103 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 102 to 103.""" @@ -296,6 +304,7 @@ def run(cls, session: Session, library_dir: Path, fmt_log): class MigrationTo104(DBMigration): version = 104 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB from DB_VERSION 103 to 104.""" @@ -328,6 +337,7 @@ def __migrate_sql_to_ts_ignore(cls, session: Session, library_dir: Path): class MigrationTo200(DBMigration): version = 200 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 200.""" @@ -421,6 +431,7 @@ class MigrationTo201(DBMigration): version = 201 initial_version = 200 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 201.""" @@ -476,6 +487,7 @@ def run(cls, session: Session, library_dir: Path, fmt_log): class MigrationTo202(DBMigration): version = 202 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): """Migrate DB to DB_VERSION 202.""" @@ -488,6 +500,7 @@ def run(cls, session: Session, library_dir: Path, fmt_log): class MigrationTo300(DBMigration): version = 300 + @override @classmethod def run(cls, session: Session, library_dir: Path, fmt_log): ## remove folder_id column from entries table From 92b40a72a9f31206578f8e0ab31031d899220ef8 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Fri, 24 Jul 2026 16:13:36 +0200 Subject: [PATCH 12/14] refactor: move set_version to DBMigrations --- src/tagstudio/core/library/alchemy/library.py | 12 ------------ src/tagstudio/core/library/alchemy/migrations.py | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 49cf80148..d582a82f7 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -1774,18 +1774,6 @@ def _get_version(engine, key: str) -> int: except Exception: return 0 - @staticmethod - def _set_version(session: Session, key: str, value: int) -> None: - """Set a version value to the DB. - - Args: - session(Session): The SQLAlchemy DB Session to use. - key(str): The key for the name of the version type to set. - value(int): The version value to set. - """ - # Insert if key has no value yet, otherwise update the value - session.merge(Version(key=key, value=value)) - def mirror_entry_fields(self, entries: list[Entry]) -> None: """Mirror fields among multiple Entry items.""" all_fields: set[BaseField] = set() diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 6685556d3..2951f02dc 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -80,7 +80,6 @@ def required(self) -> bool: return self.loaded_db_version < DB_VERSION def run(self): - from tagstudio.core.library.alchemy.library import Library # migrate DB step by step from one version to the next # (migration_method, db_version, initial_db_version) @@ -113,7 +112,7 @@ def run(self): ) self.loaded_db_version = migration.version try: - Library._set_version(session, DB_VERSION_CURRENT_KEY, migration.version) + self.__set_version(session, DB_VERSION_CURRENT_KEY, migration.version) except Exception as e: logger.info( f"[Library][Migration][{migration.version}] " @@ -130,6 +129,17 @@ def run(self): ) logger.info(f"[Library][Migration] Library migrated to DB version {DB_VERSION}") + def __set_version(self, session: Session, key: str, value: int) -> None: + """Set a version value to the DB. + + Args: + session(Session): The SQLAlchemy DB Session to use. + key(str): The key for the name of the version type to set. + value(int): The version value to set. + """ + # Insert if key has no value yet, otherwise update the value + session.merge(Version(key=key, value=value)) + class MigrationTo7(DBMigration): version = 7 From f55af0c37f265a56c7c1d0d38f43e01a509ad76d Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 27 Jul 2026 00:22:00 +0200 Subject: [PATCH 13/14] refactor: remove unnecessary assignment --- src/tagstudio/core/library/alchemy/migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 2951f02dc..69aa41636 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -34,7 +34,7 @@ class MigrationError(Exception): class DBMigration: - version: int = None # pyright: ignore[reportAssignmentType] + version: int initial_version: int | None = None @classmethod From 0a128635437dea7012d9f4e958b25606171ffb98 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 27 Jul 2026 00:35:19 +0200 Subject: [PATCH 14/14] refactor: use _ instead of __ --- src/tagstudio/core/library/alchemy/migrations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 69aa41636..94ec895ce 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -112,7 +112,7 @@ def run(self): ) self.loaded_db_version = migration.version try: - self.__set_version(session, DB_VERSION_CURRENT_KEY, migration.version) + self._set_version(session, DB_VERSION_CURRENT_KEY, migration.version) except Exception as e: logger.info( f"[Library][Migration][{migration.version}] " @@ -129,7 +129,7 @@ def run(self): ) logger.info(f"[Library][Migration] Library migrated to DB version {DB_VERSION}") - def __set_version(self, session: Session, key: str, value: int) -> None: + def _set_version(self, session: Session, key: str, value: int) -> None: """Set a version value to the DB. Args: