Refactor

mobile/lib/db/local/db.dart

@@ -9,12 +9,11 @@ import "package:photos/db/common/base.dart";
 import "package:photos/db/local/mappers.dart";
 import "package:photos/db/local/schema.dart";
 import "package:photos/log/devlog.dart";
+import 'package:photos/models/file/file.dart';
 import "package:sqlite_async/sqlite_async.dart";
 
-import 'package:photos/models/file/file.dart';
-
 class LocalDB with SqlDbBase {
-  static const _databaseName = "local_4.db";
+  static const _databaseName = "local_5.db";
   static const _batchInsertMaxCount = 1000;
   static const _smallTableBatchInsertMaxCount = 5000;
   late final SqliteDatabase _sqliteDB;
@@ -47,11 +46,11 @@ class LocalDB with SqlDbBase {
     );
   }
 
-  Future<List<AssetEntity>> getAssets() async {
+  Future<List<EnteFile>> getAssets({LocalAssertsParam? params}) async {
     final result = await _sqliteDB.execute(
-      "SELECT * FROM assets",
+      "SELECT * FROM assets ${params != null ? 'WHERE  ${params.whereClause()}' : ""}",
     );
-    return result.map((row) => LocalDBMappers.asset(row)).toList();
+    return result.map((row) => LocalDBMappers.assetRowToEnteFile(row)).toList();
   }
 
   Future<List<EnteFile>> getPathAssets(String pathID) async {
@@ -70,7 +69,7 @@ class LocalDB with SqlDbBase {
       final List<List<Object?>> values =
           slice.map((e) => LocalDBMappers.devicePathRow(e)).toList();
       await _sqliteDB.executeBatch(
-        'INSERT OR REPLACE INTO device_path ($devicePathColumns) values(${getParams(5)})',
+        'INSERT  INTO device_path ($devicePathColumns) values(${getParams(5)}) ON CONFLICT(path_id) DO UPDATE SET $updateDevicePathColumns',
         values,
       );
     });

mobile/lib/db/local/schema.dart

@@ -5,7 +5,7 @@ const assetColumns =
     "id, type, sub_type, width, height, duration_in_sec, orientation, is_fav, title, relative_path, created_at, modified_at, mime_type, latitude, longitude, scan_state";
 
 // Generate the update clause dynamically (excludes 'id')
-final updateAssetColumns = assetColumns
+final String updateAssetColumns = assetColumns
     .split(', ')
     .where((column) => column != 'id') // Exclude primary key from update
     .map((column) => '$column = excluded.$column') // Use excluded virtual table
@@ -14,6 +14,85 @@ final updateAssetColumns = assetColumns
 const devicePathColumns =
     "path_id, name, album_type, ios_album_type, ios_album_subtype";
 
+final String updateDevicePathColumns = devicePathColumns
+    .split(', ')
+    .where((column) => column != 'path_id') // Exclude primary key from update
+    .map((column) => '$column = excluded.$column') // Use excluded virtual table
+    .join(', ');
+
+const String deviceCollectionWithOneAssetQuery = '''
+WITH latest_per_path AS (
+    SELECT 
+        dpa.path_id,
+        MAX(a.created_at) as max_created
+    FROM 
+        device_path_assets dpa
+    JOIN 
+        assets a ON dpa.asset_id = a.id
+    GROUP BY 
+        dpa.path_id
+),
+ranked_assets AS (
+    SELECT 
+        dpa.path_id,
+        a.*,
+        ROW_NUMBER() OVER (PARTITION BY dpa.path_id ORDER BY a.id) as rn
+    FROM 
+        device_path_assets dpa
+    JOIN 
+        assets a ON dpa.asset_id = a.id
+    JOIN 
+        latest_per_path lpp ON dpa.path_id = lpp.path_id AND a.created_at = lpp.max_created
+)
+SELECT 
+    dp.*,
+    ra.*
+FROM 
+    device_path dp
+JOIN 
+    ranked_assets ra ON dp.path_id = ra.path_id AND ra.rn = 1
+    ''';
+
+class LocalAssertsParam {
+  int? limit;
+  int? offset;
+  String? orderByColumn;
+  bool? isAsc;
+  (int?, int?)? createAtRange;
+
+  LocalAssertsParam({
+    this.limit,
+    this.offset,
+    this.orderByColumn = "created_at",
+    this.isAsc = false,
+    this.createAtRange,
+  });
+
+  String get orderBy => orderByColumn == null
+      ? ""
+      : "ORDER BY $orderByColumn ${isAsc! ? "ASC" : "DESC"}";
+
+  String get limitOffset => (limit != null && offset != null)
+      ? "LIMIT $limit +  OFFSET $offset)"
+      : (limit != null)
+          ? "LIMIT $limit"
+          : "";
+
+  String get createAtRangeStr => (createAtRange == null ||
+          createAtRange!.$1 == null)
+      ? ""
+      : "(created_at BETWEEN ${createAtRange!.$1} AND ${createAtRange!.$2})";
+
+  String whereClause() {
+    final where = <String>[];
+    if (createAtRangeStr.isNotEmpty) {
+      where.add(createAtRangeStr);
+    }
+
+    return (where.isEmpty ? "" : where.join(" AND ")) + " " + limitOffset;
+  }
+}
+
 class LocalDBMigration {
   static const migrationScripts = [
     '''
@@ -63,7 +142,10 @@ class LocalDBMigration {
       name TEXT NOT NULL,
       PRIMARY KEY (id, name)
     );
+    ''',
     '''
+        CREATE INDEX IF NOT EXISTS assets_created_at ON assets(created_at);
+    ''',
   ];
 
   static Future<void> migrate(

mobile/lib/services/local/device_albums.dart

@@ -16,7 +16,7 @@ extension DeviceAlbums on LocalImportService {
             path.id,
             path.name,
             count: cache.pathToAssetIDs[path.id]?.length ?? 0,
-            thumbnail: EnteFile.fromAssetSync(asset),
+            thumbnail: asset,
           ),
         );
       }

mobile/lib/services/local/local_assets_cache.dart

@@ -1,15 +1,18 @@
 import "package:photo_manager/photo_manager.dart";
+import "package:photos/models/file/file.dart";
 import "package:photos/services/local/import/model.dart";
 
 class LocalAssetsCache {
   final Map<String, AssetPathEntity> assetPaths;
-  final Map<String, AssetEntity> assets;
+  final Map<String, EnteFile> assets;
   final Map<String, Set<String>> pathToAssetIDs;
+  final List<EnteFile> sortedAssets;
 
   LocalAssetsCache({
     required this.assetPaths,
     required this.assets,
     required this.pathToAssetIDs,
+    required this.sortedAssets,
   });
 
   void updateForDiff({
@@ -18,7 +21,7 @@ class LocalAssetsCache {
   }) {
     if (incrementalDiff != null) {
       for (final asset in incrementalDiff.assets) {
-        assets[asset.id] = asset;
+        assets[asset.id] = EnteFile.fromAssetSync(asset);
       }
       for (final path in incrementalDiff.addedOrModifiedPaths) {
         assetPaths[path.id] = path;
@@ -35,7 +38,7 @@ class LocalAssetsCache {
         assetPaths.remove(id);
       }
       for (final asset in fullDiff.missingAssetsInApp) {
-        assets[asset.id] = asset;
+        assets[asset.id] = EnteFile.fromAssetSync(asset);
       }
       for (final entry in fullDiff.updatePathToLocalIDs.entries) {
         // delete old mappings
@@ -44,16 +47,15 @@ class LocalAssetsCache {
     }
   }
 
-  Map<String, AssetEntity> getPathToLatestAsset() {
-    final Map<String, AssetEntity> pathToLatestAsset = {};
+  Map<String, EnteFile> getPathToLatestAsset() {
+    final Map<String, EnteFile> pathToLatestAsset = {};
     for (final entry in pathToAssetIDs.entries) {
-      AssetEntity? latestAsset;
+      EnteFile? latestAsset;
       for (final id in entry.value) {
         final asset = assets[id];
         if (asset != null &&
             (latestAsset == null ||
-                (asset.createDateSecond ?? 0) >
-                    (latestAsset.createDateSecond ?? 0))) {
+                (asset.creationTime ?? 0) > (latestAsset.creationTime ?? 0))) {
           latestAsset = asset;
         }
       }

mobile/lib/services/local/local_import.dart

@@ -8,6 +8,7 @@ import "package:photos/events/local_photos_updated_event.dart";
 import "package:photos/events/permission_granted_event.dart";
 import 'package:photos/events/sync_status_update_event.dart';
 import 'package:photos/extensions/stop_watch.dart';
+import "package:photos/models/file/file.dart";
 import "package:photos/service_locator.dart";
 import 'package:photos/services/app_lifecycle_service.dart';
 import "package:photos/services/local/import/device_assets.service.dart";
@@ -31,8 +32,8 @@ class LocalImportService {
   );
   final Lock _lock = Lock();
 
-  static const lastLocalDBSyncTime = "localImport.lastSyncTime";
-  static const kHasCompletedFirstImportKey = "has_completed_firstImport_x";
+  static const lastLocalDBSyncTime = "localImport.lastSyncTime_2";
+  static const kHasCompletedFirstImportKey = "has_completed_firstImport_2";
 
   LocalImportService._privateConstructor();
 
@@ -168,13 +169,15 @@ class LocalImportService {
           if (_localAssetsCache == null) {
             _log.info("loading local assets cache");
             final List<AssetPathEntity> paths = await localDB.getAssetPaths();
-            final List<AssetEntity> assets = await localDB.getAssets();
+            final List<EnteFile> assets = await localDB.getAssets();
             final Map<String, Set<String>> pathToAssetIDs =
                 await localDB.pathToAssetIDs();
             _localAssetsCache = LocalAssetsCache(
               assetPaths: Map.fromEntries(paths.map((e) => MapEntry(e.id, e))),
-              assets: Map.fromEntries(assets.map((e) => MapEntry(e.id, e))),
+              assets:
+                  Map.fromEntries(assets.map((e) => MapEntry(e.localID!, e))),
               pathToAssetIDs: pathToAssetIDs,
+              sortedAssets: assets,
             );
           }
         },
@@ -191,8 +194,6 @@ class LocalImportService {
     return _localAssetsCache!;
   }
 
-  LocalAssetsCache? get localAssetsCache => _localAssetsCache;
-
   Lock getLock() {
     return _lock;
   }

mobile/lib/ui/home/home_gallery_widget_v2.dart

@@ -3,17 +3,16 @@ import "dart:async";
 import 'package:flutter/material.dart';
 import "package:logging/logging.dart";
 import 'package:photos/core/event_bus.dart';
+import "package:photos/db/local/schema.dart";
 import 'package:photos/events/backup_folders_updated_event.dart';
 import 'package:photos/events/files_updated_event.dart';
 import 'package:photos/events/force_reload_home_gallery_event.dart';
 import "package:photos/events/hide_shared_items_from_home_gallery_event.dart";
 import 'package:photos/events/local_photos_updated_event.dart';
-import "package:photos/models/file/file.dart";
 import 'package:photos/models/file_load_result.dart';
 import 'package:photos/models/gallery_type.dart';
 import 'package:photos/models/selected_files.dart';
 import "package:photos/service_locator.dart";
-import "package:photos/services/local/local_import.dart";
 import 'package:photos/ui/viewer/actions/file_selection_overlay_bar.dart';
 import 'package:photos/ui/viewer/gallery/gallery.dart';
 import "package:photos/ui/viewer/gallery/state/gallery_files_inherited_widget.dart";
@@ -77,19 +76,21 @@ class _HomeGalleryWidgetV2State extends State<HomeGalleryWidgetV2> {
       key: ValueKey(_shouldHideSharedItems),
       asyncLoader: (creationStartTime, creationEndTime, {limit, asc}) async {
         Logger("_HomeGalleryWidgetV2State").info("Loading home gallery files");
-        final cache = LocalImportService.instance.localAssetsCache ??
-            await LocalImportService.instance.getLocalAssetsCache();
-        final enteFiles = <EnteFile>[];
-        for (var asset in cache.assets.values) {
-          enteFiles.add(EnteFile.fromAssetSync(asset));
-        }
-        enteFiles.sort(
-          (a, b) => (a.creationTime ?? 0).compareTo(b.creationTime ?? 0),
+        final enteFiles = await localDB.getAssets(
+          params: LocalAssertsParam(
+            limit: limit,
+            isAsc: asc ?? false,
+            createAtRange: (creationStartTime, creationEndTime),
+          ),
         );
+
         Logger("_HomeGalleryWidgetV2State").info(
           "Load home gallery files ${enteFiles.length} files",
         );
-        return FileLoadResult(enteFiles, false);
+        return FileLoadResult(
+          enteFiles,
+          limit != null && enteFiles.length <= limit,
+        );
       },
       reloadEvent: Bus.instance.on<LocalPhotosUpdatedEvent>(),
       removalEventTypes: const {

mobile/lib/ui/settings/backup/backup_folder_selection_page_v2.dart

@@ -41,7 +41,7 @@ class _BackupFolderSelectionPageV2State
   final Set<String> _selectedDevicePathIDs = <String>{};
   List<AssetPathEntity>? _assetPathEntities;
   Map<String, Set<String>> _assetCount = {};
-  final Map<String, AssetEntity> _pathToLatestAsset = {};
+  final Map<String, EnteFile> _pathToLatestAsset = {};
 
   @override
   void initState() {
@@ -54,15 +54,15 @@ class _BackupFolderSelectionPageV2State
       _assetPathEntities!.removeWhere(
         (path) => (_assetCount[path.id] ?? {}).isEmpty,
       );
-      final List<AssetEntity> latestAssets = c.assets.values.toList();
+      final List<EnteFile> latestAssets = c.assets.values.toList();
       for (final path in _assetPathEntities ?? []) {
         final assetIDs = _assetCount[path.id] ?? {};
         for (final sortedAsset in latestAssets) {
-          if (assetIDs.contains(sortedAsset.id)) {
+          if (assetIDs.contains(sortedAsset.localID!)) {
             if (_pathToLatestAsset.containsKey(path.id)) {
               // check time and insert one with latest time
-              if (_pathToLatestAsset[path.id]!.createDateSecond! <
-                  sortedAsset.createDateSecond!) {
+              if (_pathToLatestAsset[path.id]!.creationTime! <
+                  sortedAsset.creationTime!) {
                 _pathToLatestAsset[path.id] = sortedAsset;
               }
             } else {
@@ -449,9 +449,7 @@ class _BackupFolderSelectionPageV2State
 
   // todo: replace with asset thumbnail provider
   Widget _getThumbnail(AssetPathEntity path, bool isSelected) {
-    final file = _pathToLatestAsset[path.id] != null
-        ? EnteFile.fromAssetSync(_pathToLatestAsset[path.id]!)
-        : null;
+    final file = _pathToLatestAsset[path.id];
     return ClipRRect(
       borderRadius: BorderRadius.circular(8),
       child: SizedBox(