feat!: add file date metadata to db

Author: CyanVoxel

src/tagstudio/core/library/alchemy/library.py

@@ -7,6 +7,7 @@
 # pyright: reportDeprecated=false
 
 
+import platform
 import re
 import shutil
 import sys
@@ -93,6 +94,7 @@
     TextFieldTemplate,
 )
 from tagstudio.core.library.alchemy.joins import TagEntry, TagParent
+from tagstudio.core.library.alchemy.metadata import FileMetadata
 from tagstudio.core.library.alchemy.models import (
     Entry,
     Folder,
@@ -105,6 +107,7 @@
 from tagstudio.core.library.alchemy.visitors import SQLBoolExpressionBuilder
 from tagstudio.core.library.ignore import migrate_ext_list
 from tagstudio.core.library.json.library import Library as JsonLibrary
+from tagstudio.core.utils.stat import get_date_created, get_date_modified
 from tagstudio.core.utils.types import unwrap
 from tagstudio.qt.translations import Translations
 
@@ -924,7 +927,11 @@ def get_entry(self, entry_id: int) -> Entry | None:
             return entry
 
     def get_entry_full(
-        self, entry_id: int, with_fields: bool = True, with_tags: bool = True
+        self,
+        entry_id: int,
+        with_fields: bool = True,
+        with_tags: bool = True,
+        with_metadata: bool = True,
     ) -> Entry | None:
         """Load entry and join with all joins and all tags."""
         # NOTE: TODO: Currently this method makes multiple separate queries to the db and combines
@@ -954,6 +961,11 @@ def get_entry_full(
                     )
                 )
 
+            if with_metadata:
+                entry_stmt = entry_stmt.outerjoin(Entry.file_metadata).options(
+                    selectinload(Entry.file_metadata),
+                )
+
             start_time = time.time()
             entry = session.scalar(entry_stmt)
             if with_tags:
@@ -1147,10 +1159,79 @@ def remove_entries(self, entry_ids: list[int]) -> None:
                 session.query(Entry).where(Entry.id.in_(sub_list)).delete()
             session.commit()
 
-    def has_entry_with_path(self, path: Path) -> bool:
-        """Check if an entry with this path is in the library."""
+    def get_entry_id_from_path(self, path: Path) -> int:
+        """Attempt to return an Entry ID given a filepath, else return -1."""
         with Session(self.engine) as session:
-            return session.query(exists().where(Entry.path == path)).scalar()
+            return session.scalar(select(Entry.id).where(Entry.path == path).limit(1)) or -1
+
+    # def update_entry_file_metadata(
+    #     self, entry_id: int, date_created: datetime | None, date_modified: datetime | None
+    # ):
+    #     with Session(self.engine) as session:
+    #         stmt = update(FileMetadata).where(
+    #             and_(
+    #                 FileMetadata.entry_id == entry_id,
+    #             )
+    #         )
+    #         if date_created:
+    #             stmt = stmt.values(date_created=date_created)
+    #         if date_modified:
+    #             stmt = stmt.values(date_modified=date_modified)
+
+    #         session.execute(stmt)
+    #         session.commit()
+
+    def refresh_file_entry_stats(self, entry_id: int, path: Path | None):
+        """Updates a file entry's associated stat() data."""
+        needs_update = False
+
+        entry = self.get_entry_full(
+            entry_id, with_fields=False, with_tags=False, with_metadata=True
+        )
+        if not entry:
+            return
+
+        if not path:
+            full_path = unwrap(self.library_dir) / entry.path
+        else:
+            full_path = unwrap(self.library_dir) / path
+
+        logger.info(full_path)
+
+        file_date_created = get_date_created(full_path)
+        file_date_modified = get_date_modified(full_path)
+
+        # Log info
+        if entry.date_created != file_date_created:
+            logger.info(f"Difference in date_created!: {entry.date_created}/{file_date_created}")
+            needs_update = True
+        else:
+            logger.info("No difference in date_created.")
+
+        if entry.date_modified != file_date_modified:
+            logger.info(f"Difference in date_modified!: {entry.date_modified}/{file_date_modified}")
+            needs_update = True
+        else:
+            logger.info("No difference in date_modified")
+
+        if needs_update:
+            return
+        else:
+            logger.info(f"Updating entry file_metadata for {full_path}")
+
+        with Session(self.engine) as session:
+            stmt = update(FileMetadata).where(
+                and_(
+                    FileMetadata.entry_id == entry_id,
+                )
+            )
+            if file_date_created:
+                stmt = stmt.values(date_created=file_date_created)
+            if file_date_modified:
+                stmt = stmt.values(date_modified=file_date_modified)
+
+            session.execute(stmt)
+            session.commit()
 
     def get_paths(self, limit: int = -1) -> list[str]:
         path_strings: list[str] = []
@@ -1468,7 +1549,7 @@ def update_entry_path(self, entry_id: int | Entry, path: Path) -> bool:
 
         Returns True if the action succeeded and False if the path already exists.
         """
-        if self.has_entry_with_path(path):
+        if self.get_entry_id_from_path(path) >= 0:
             return False
         if isinstance(entry_id, Entry):
             entry_id = entry_id.id

src/tagstudio/core/library/alchemy/metadata.py

@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: (c) TagStudio Contributors
+# SPDX-License-Identifier: MIT
+
+
+from __future__ import annotations
+
+from datetime import datetime as dt
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, override
+
+from sqlalchemy import ForeignKey, ForeignKeyConstraint, Integer, null
+from sqlalchemy.orm import Mapped, declared_attr, mapped_column, relationship
+
+
+from tagstudio.core.library.alchemy.db import Base, PathType
+
+from tagstudio.core.library.alchemy.joins import TagParent
+
+if TYPE_CHECKING:
+    from tagstudio.core.library.alchemy.models import Entry
+
+
+class FileMetadata(Base):
+    """Table that includes file data and metadata obtained from os.stat() for entries."""
+
+    __tablename__ = "file_metadata"
+
+    entry_id: Mapped[int] = mapped_column(
+        ForeignKey("entries.id"), primary_key=True, nullable=False
+    )
+
+    # NOTE: These dates are stored as floats because that's their natural form from os.stat()
+    # and comparisons are quicker without having to convert to/from datetime objects.
+    date_created: Mapped[float | None]
+    date_modified: Mapped[float | None]
+
+    def __init__(
+        self,
+        entry_id: int,
+        date_created: float | None = None,
+        date_modified: float | None = None,
+    ) -> None:
+        super().__init__()
+        self.entry_id = entry_id
+
+        # # Path data
+        # self.path = path
+        # self.filename = path.name
+        # self.suffix = path.suffix.lstrip(".").lower()
+
+        # File metadata
+        self.date_created = date_created  # st_birthtime on Windows and Mac, st_ctime on Linux
+        self.date_modified = date_modified  # st_mtime
+
+
+class ExifMetadata(Base):
+    """Contains Exif metadata for a entries."""
+
+    __tablename__ = "exif_metadata"
+
+    entry_id: Mapped[int] = mapped_column(
+        ForeignKey("entries.id"), primary_key=True, nullable=False
+    )
+    date_taken: Mapped[dt | None]
+
+    def __init__(
+        self,
+        entry_id: int,
+        date_taken: dt | None = None,
+    ) -> None:
+        super().__init__()
+        self.entry_id = entry_id
+        self.date_taken = date_taken  # Exif.Image.DateTime
+
+
+class DimensionMetadata(Base):
+    """Contains dimension metadata for entries (e.g. image and video files)."""
+
+    __tablename__ = "dimension_metadata"
+
+    entry_id: Mapped[int] = mapped_column(
+        ForeignKey("entries.id"), primary_key=True, nullable=False
+    )
+    width: Mapped[int] = mapped_column(nullable=False)
+    height: Mapped[int] = mapped_column(nullable=False)
+
+    def __init__(
+        self,
+        entry_id: int,
+        width: int,
+        height: int,
+    ) -> None:
+        super().__init__()
+        self.entry_id = entry_id
+        self.width = width
+        self.height = height
+
+
+class DurationMetadata(Base):
+    """Contains duration metadata for entries (e.g. audio and video files)."""
+
+    __tablename__ = "duration_metadata"
+
+    entry_id: Mapped[int] = mapped_column(
+        ForeignKey("entries.id"), primary_key=True, nullable=False
+    )
+    duration: Mapped[float] = mapped_column(nullable=False)
+
+    def __init__(
+        self,
+        entry_id: int,
+        duration: float,
+    ) -> None:
+        super().__init__()
+        self.entry_id = entry_id
+        self.duration = duration

src/tagstudio/core/library/alchemy/models.py

@@ -17,6 +17,8 @@
     TextField,
 )
 from tagstudio.core.library.alchemy.joins import TagParent
+from tagstudio.core.library.alchemy.metadata import FileMetadata
+from tagstudio.core.utils.stat import get_date_created, get_date_modified
 
 
 class Namespace(Base):
@@ -182,6 +184,7 @@ def __ge__(self, other: "Tag") -> bool:
         return self.name >= other.name
 
 
+# TODO: Use or replace these with an actual multi-root implementation
 class Folder(Base):
     __tablename__ = "folders"
 
@@ -196,15 +199,16 @@ class Entry(Base):
 
     id: Mapped[int] = mapped_column(primary_key=True)
 
+    # TODO: Use or replace these with an actual multi-root implementation
     folder_id: Mapped[int] = mapped_column(ForeignKey("folders.id"))
     folder: Mapped[Folder] = relationship("Folder")
 
+    # TODO: Possibly move to FileMetadata table if Entry is split into Entry/FileEntry (see #588)
     path: Mapped[Path] = mapped_column(PathType, unique=True)
     filename: Mapped[str] = mapped_column()
     suffix: Mapped[str] = mapped_column()
-    date_created: Mapped[dt | None]
-    date_modified: Mapped[dt | None]
-    date_added: Mapped[dt | None]
+
+    date_added: Mapped[dt | None]  # The date this entry was added to the library
 
     tags: Mapped[set[Tag]] = relationship(secondary="tag_entries")
 
@@ -217,6 +221,11 @@ class Entry(Base):
         cascade="all, delete",
     )
 
+    file_metadata: Mapped["FileMetadata"] = relationship(
+        uselist=False,
+        cascade="all, delete-orphan",
+    )
+
     @property
     def fields(self) -> list[BaseField]:
         fields: list[BaseField] = []
@@ -232,30 +241,35 @@ def is_favorite(self) -> bool:
     def is_archived(self) -> bool:
         return any(tag.id == TAG_ARCHIVED for tag in self.tags)
 
+    @property
+    def date_created(self) -> float | None:
+        return self.file_metadata.date_created if self.file_metadata else None
+
+    @property
+    def date_modified(self) -> float | None:
+        return self.file_metadata.date_modified if self.file_metadata else None
+
     def __init__(
         self,
         path: Path,
         folder: Folder,
         fields: list[BaseField],
         id: int | None = None,
-        date_created: dt | None = None,
-        date_modified: dt | None = None,
         date_added: dt | None = None,
+        # date_created: float | None = None,
+        # date_modified: float | None = None,
+        path_for_file_metadata: Path | None = None,
     ) -> None:
         super().__init__()
-        self.path = path
-        self.folder = folder
+
         self.id = id  # pyright: ignore[reportAttributeAccessIssue]
+
+        self.folder = folder  # NOTE: Currently unused
+        self.path = path
         self.filename = path.name
         self.suffix = path.suffix.lstrip(".").lower()
 
-        # The date the file associated with this entry was created.
-        # st_birthtime on Windows and Mac, st_ctime on Linux.
-        self.date_created = date_created
-        # The date the file associated with this entry was last modified: st_mtime.
-        self.date_modified = date_modified
-        # The date this entry was added to the library.
-        self.date_added = date_added
+        self.date_added = date_added  # The date this entry was added to the library
 
         for field in fields:
             if isinstance(field, TextField):
@@ -265,6 +279,13 @@ def __init__(
             else:
                 raise ValueError(f"Invalid field type: {field}")
 
+        if path_for_file_metadata:
+            self.file_metadata = FileMetadata(
+                entry_id=self.id,
+                date_created=get_date_created(path_for_file_metadata),
+                date_modified=get_date_modified(path_for_file_metadata),
+            )
+
     def has_tag(self, tag: Tag) -> bool:
         return tag in self.tags