Compare commits

...

75 Commits

Author SHA1 Message Date
laurenspriem
a694bf9b6c flutter downgrade dependencies 2025-07-24 09:02:05 +02:00
laurenspriem
bfcfa691a2 upgrade frb and run frb gen command 2025-07-23 17:55:07 +02:00
laurenspriem
2703c6a33a Bump to be up to date with internal branch 2025-07-23 17:39:19 +02:00
laurenspriem
3ad94f362a Resolve merge conflicts 2025-07-23 17:06:54 +02:00
laurenspriem
8508ca74f2 Merge branch 'main' into usearch_again 2025-07-23 15:05:06 +02:00
laurenspriem
eed12c2089 Merge branch 'internal-15_06_2025' into usearch_again 2025-06-09 12:34:55 +05:30
laurenspriem
889aed6024 Bump for internal release 2025-06-09 12:34:29 +05:30
laurenspriem
ac7840cbfd Merge branch 'internal-15_06_2025' into usearch_again 2025-06-09 12:33:03 +05:30
laurenspriem
1f1304ca5b Upgrade usearch to fix Armv8-R issues 2025-06-09 12:31:38 +05:30
Neeraj Gupta
94098d8a07 Bump version 2025-06-06 12:35:30 +05:30
Neeraj Gupta
4b9c5fcb73 Merge branch 'internal-15_06_2025' of https://github.com/ente-io/auth into internal-15_06_2025 2025-06-06 12:33:48 +05:30
Neeraj Gupta
6ed16e5e02 Merge branch 'main' into internal-15_06_2025 2025-06-06 12:33:21 +05:30
laurenspriem
82a8e504af Merge branch 'internal-15_06_2025' into usearch_again 2025-06-04 22:14:26 +05:30
ashilkn
cc1660d9af bump up build number" 2025-06-04 18:24:59 +05:30
ashilkn
52b6fc108b Merge branch 'memory_improvement' into internal-15_06_2025 2025-06-04 18:19:37 +05:30
Neeraj Gupta
8b3b20aa93 Remove unsued type 2025-06-02 18:38:45 +05:30
Neeraj Gupta
408b0bfe2d Merge branch 'internal-15_06_2025' of https://github.com/ente-io/auth into internal-15_06_2025 2025-06-02 17:55:33 +05:30
Neeraj Gupta
655be76428 Bump version 2025-06-02 17:55:06 +05:30
Neeraj Gupta
9fedf8d6b7 Merge branch 'main' into internal-15_06_2025 2025-06-02 17:54:47 +05:30
ashilkn
7c4e775872 Bump build number 2025-06-02 14:56:04 +05:30
Neeraj Gupta
ecfa640c28 Bump version 2025-05-30 15:44:46 +05:30
Neeraj Gupta
1997eb20f3 Merge branch 'main' into internal-15_06_2025 2025-05-30 15:44:30 +05:30
laurenspriem
726425bbb6 Put vector db behind feature flag internal 2025-05-27 14:07:34 +05:30
laurenspriem
c068f26604 Aggressive logging of vectorDB migration 2025-05-22 11:32:36 +05:30
laurenspriem
e60c2b1192 GC hints 2025-05-22 11:19:19 +05:30
laurenspriem
beb049f817 Reduce batch size in migration 2025-05-22 10:38:43 +05:30
laurenspriem
7021c9fe02 Bump for internal release 2025-05-12 17:05:31 +05:30
laurenspriem
c2d5dece9e Merge branch 'main' into usearch_again 2025-05-12 17:04:42 +05:30
laurenspriem
b76d41b84d Specify rust version in readme 2025-05-12 15:48:27 +05:30
laurenspriem
3b9c76649d Update readme to include rust 2025-05-12 15:12:31 +05:30
laurenspriem
62ed8b6975 Log vector db stats when opening connection 2025-05-12 14:46:55 +05:30
laurenspriem
2422dba4d4 vector db more stats logging 2025-05-12 14:23:16 +05:30
laurenspriem
eb1916e3a3 integrate vector db in magic search 2025-05-12 11:39:02 +05:30
laurenspriem
df0d9137a6 Integration clip embeddings in vector db 2025-05-11 13:09:56 +05:30
laurenspriem
fc36b87965 Clip migration method 2025-05-11 13:09:20 +05:30
laurenspriem
63d90ea275 Class for vector db stats 2025-05-09 16:36:39 +05:30
laurenspriem
bb7f8a5eef More testing 2025-05-09 15:59:46 +05:30
laurenspriem
2f5a02ec43 delete table option 2025-05-09 12:57:44 +05:30
laurenspriem
d411d91966 vector db api ensure capacity safety 2025-05-09 12:56:59 +05:30
laurenspriem
54b712953a vector db api let clear include capacity reset 2025-05-09 10:49:03 +05:30
laurenspriem
27ad020adc Testing clip migration to vector DB 2025-05-08 17:40:01 +05:30
laurenspriem
ce112bd4d7 Index stats method 2025-05-08 15:23:23 +05:30
laurenspriem
2ffb73d053 Consistent method parameters 2025-05-08 15:07:50 +05:30
laurenspriem
6478d438b5 vector db api ensure never duplicate keys 2025-05-08 14:30:51 +05:30
laurenspriem
d87069eb4c vectordb api add documentation 2025-05-08 12:31:09 +05:30
laurenspriem
5447350ab1 vector db api add check for key 2025-05-08 12:29:41 +05:30
laurenspriem
ea1a2960bf First implementation of clip vector db 2025-05-08 12:08:55 +05:30
laurenspriem
832f2c451e Add bulk get method to vector db api 2025-05-08 11:47:50 +05:30
laurenspriem
715c7c23a7 Add bulk remove embeddings api 2025-05-08 10:29:25 +05:30
laurenspriem
e9c2e40a43 Update to latest usearch 2025-05-07 13:25:40 +05:30
laurenspriem
603c275c09 Update basic usearch test 2025-05-07 12:01:45 +05:30
laurenspriem
7b9d6df2fd Fix ios build issue 2025-05-07 11:32:53 +05:30
laurenspriem
a4afecef3d Fix ios config 2025-05-07 10:50:39 +05:30
laurenspriem
4d9bfb89ae macos config 2025-05-07 10:36:17 +05:30
laurenspriem
f2a74bd35e Merge branch 'main' into usearch_again 2025-05-06 15:34:59 +05:30
laurenspriem
8c65a21b86 don't generate for web 2025-04-10 13:03:52 +05:30
laurenspriem
a07e8477fb format 2025-04-09 15:34:06 +05:30
laurenspriem
8b489e9ced Give distances in bulk search 2025-04-09 15:31:03 +05:30
laurenspriem
77e2bb1d46 Stop our own vector comparisons in benchmark 2025-04-09 15:21:20 +05:30
laurenspriem
4ce24e080a logging of benchmarking 2025-04-09 14:22:39 +05:30
laurenspriem
4e5ca3dca6 Benchmark face embeddings 2025-04-09 13:43:39 +05:30
laurenspriem
2ed155ab47 ignore trailing commas in generated code 2025-04-09 13:14:26 +05:30
laurenspriem
65e71e3caf Reintroduce reset_index method 2025-04-09 10:49:50 +05:30
laurenspriem
ee5efbcfcc Don't consume index if not needed 2025-04-09 10:43:07 +05:30
laurenspriem
6cf4530f7d Remove reset index 2025-04-09 10:06:46 +05:30
laurenspriem
e6ee09ca30 Back to basic error handling 2025-04-08 17:03:41 +05:30
laurenspriem
6d2f53b86c Update usearch 2025-04-08 14:56:47 +05:30
laurenspriem
6500748c5a Make vector db stateful in rust 2025-04-08 14:48:30 +05:30
laurenspriem
120dbeb4fc Fix null pointer crash 2025-04-05 16:56:14 +05:30
laurenspriem
c42807487b Upgrade Android NDK to r28 latest stable 2025-04-05 16:11:03 +05:30
laurenspriem
e707e24da9 first integration of usearch 2025-04-05 16:10:39 +05:30
laurenspriem
af817ec439 Test rust 2025-04-04 11:49:54 +05:30
laurenspriem
ddb44d8fd7 Integrate flutter_rust_bridge 2025-03-31 15:56:03 +05:30
laurenspriem
778822b12d Upgrade sdk 2025-03-31 15:47:51 +05:30
laurenspriem
9599ec3236 dart format 2025-03-31 15:34:48 +05:30
121 changed files with 8984 additions and 261 deletions

Submodule auth/assets/simple-icons added at 6dcfdc2f58

1
auth/flutter Submodule

Submodule auth/flutter added at 5874a72aa4

View File

@@ -46,25 +46,27 @@ You can alternatively install the build from PlayStore or F-Droid.
## 🧑‍💻 Building from source
1. [Install Flutter v3.24.3](https://flutter.dev/docs/get-started/install).
1. Install [Flutter v3.24.3](https://flutter.dev/docs/get-started/install) and [Rust v1.85.1](https://www.rust-lang.org/tools/install).
2. Pull in all submodules with `git submodule update --init --recursive`
2. Install [Flutter Rust Bridge](https://cjycode.com/flutter_rust_bridge/) with `cargo install flutter_rust_bridge_codegen`
3. Enable repo git hooks `git config core.hooksPath hooks`
3. Pull in all submodules with `git submodule update --init --recursive`
4. If using Visual Studio Code, add the [Flutter
4. Enable repo git hooks `git config core.hooksPath hooks`
5. If using Visual Studio Code, add the [Flutter
Intl](https://marketplace.visualstudio.com/items?itemName=localizely.flutter-intl)
extension
5. On Android:
6. On Android:
* For development, run `flutter run -t lib/main.dart --flavor independent`
- For development, run `flutter run -t lib/main.dart --flavor independent`
* For building APK, [setup your
- For building APK, [setup your
keystore](https://docs.flutter.dev/deployment/android#create-an-upload-keystore)
and run `flutter build apk --release --flavor independent`
6. For iOS, run `flutter build ios`
7. For iOS, run `flutter build ios`
Some common issues and troubleshooting tips are in [docs/dev](docs/dev.md).
@@ -88,11 +90,12 @@ issue](https://github.com/ente-io/ente/issues/new?title=Request+for+New+Language
to have it added.
## Certificate Fingerprints
- **SHA1**: E1:60:10:18:B6:B0:2E:A3:74:6F:90:67:50:30:29:75:0E:EF:6D:39
- **SHA256**: 35:ED:56:81:B7:0B:B3:BD:35:D9:0D:85:6A:F5:69:4C:50:4D:EF:46:AA:D8:3F:77:7B:1C:67:5C:F4:51:35:0B
To verify these fingerprints, use the following command:
```bash
apksigner verify --print-certs <path_to_apk>
```

View File

@@ -31,7 +31,7 @@ if (keystorePropertiesFile.exists()) {
android {
namespace = "io.ente.photos"
compileSdk = 35
ndkVersion = flutter.ndkVersion
ndkVersion = "28.0.13004108"
compileOptions {
sourceCompatibility = JavaVersion.VERSION_1_8

View File

@@ -0,0 +1,8 @@
rust_input: crate::api
rust_root: rust/
dart_output: lib/src/rust
dart_preamble: |
// ignore_for_file: require_trailing_commas
web: false

View File

@@ -91,10 +91,9 @@ Future<void> dismissUpdateAppDialog(WidgetTester tester) async {
await tester.pumpAndSettle();
}
///Use this widget as floating action buttom in HomeWidget so that frames
///are built and rendered continuously so that timeline trace has continuous
///data. Change the duraiton in `_startTimer()` to control the duraiton of
///are built and rendered continuously so that timeline trace has continuous
///data. Change the duraiton in `_startTimer()` to control the duraiton of
///test on app init.
// class TempWidget extends StatefulWidget {
@@ -127,4 +126,4 @@ Future<void> dismissUpdateAppDialog(WidgetTester tester) async {
// ? const CircularProgressIndicator()
// : const SizedBox.shrink();
// }
// }
// }

View File

@@ -0,0 +1,13 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:integration_test/integration_test.dart';
import "package:photos/src/rust/api/simple.dart";
import 'package:photos/src/rust/frb_generated.dart';
void main() {
IntegrationTestWidgetsFlutterBinding.ensureInitialized();
setUpAll(() async => await RustLib.init());
testWidgets('Can call rust function', (WidgetTester tester) async {
final testString = greet(name: "Tom");
expect(testString.contains('Tom'), true);
});
}

View File

@@ -288,6 +288,7 @@ DEPENDENCIES:
- photo_manager (from `.symlinks/plugins/photo_manager/ios`)
- privacy_screen (from `.symlinks/plugins/privacy_screen/ios`)
- receive_sharing_intent (from `.symlinks/plugins/receive_sharing_intent/ios`)
- rust_lib_photos (from `.symlinks/plugins/rust_lib_photos/ios`)
- sentry_flutter (from `.symlinks/plugins/sentry_flutter/ios`)
- share_plus (from `.symlinks/plugins/share_plus/ios`)
- shared_preferences_foundation (from `.symlinks/plugins/shared_preferences_foundation/darwin`)
@@ -415,6 +416,8 @@ EXTERNAL SOURCES:
:path: ".symlinks/plugins/privacy_screen/ios"
receive_sharing_intent:
:path: ".symlinks/plugins/receive_sharing_intent/ios"
rust_lib_photos:
:path: ".symlinks/plugins/rust_lib_photos/ios"
sentry_flutter:
:path: ".symlinks/plugins/sentry_flutter/ios"
share_plus:

View File

@@ -566,6 +566,7 @@
"${BUILT_PRODUCTS_DIR}/photo_manager/photo_manager.framework",
"${BUILT_PRODUCTS_DIR}/privacy_screen/privacy_screen.framework",
"${BUILT_PRODUCTS_DIR}/receive_sharing_intent/receive_sharing_intent.framework",
"${BUILT_PRODUCTS_DIR}/rust_lib_photos/rust_lib_photos.framework",
"${BUILT_PRODUCTS_DIR}/sentry_flutter/sentry_flutter.framework",
"${BUILT_PRODUCTS_DIR}/share_plus/share_plus.framework",
"${BUILT_PRODUCTS_DIR}/shared_preferences_foundation/shared_preferences_foundation.framework",
@@ -662,6 +663,7 @@
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/photo_manager.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/privacy_screen.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/receive_sharing_intent.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/rust_lib_photos.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/sentry_flutter.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/share_plus.framework",
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/shared_preferences_foundation.framework",

View File

@@ -1,4 +1,3 @@
import "dart:io";
import 'package:photos/core/cache/lru_map.dart';

View File

@@ -58,7 +58,7 @@ bool isHandledSyncError(Object errObj) {
class LockAlreadyAcquiredError extends Error {}
class LockFreedError extends Error{}
class LockFreedError extends Error {}
class UnauthorizedError extends Error {}

View File

@@ -0,0 +1,238 @@
import "dart:typed_data" show Float32List;
import "package:flutter_rust_bridge/flutter_rust_bridge.dart" show Uint64List;
import "package:logging/logging.dart";
import "package:path/path.dart";
import "package:path_provider/path_provider.dart";
import "package:photos/models/ml/vector.dart";
import "package:photos/services/machine_learning/semantic_search/query_result.dart";
import "package:photos/src/rust/api/usearch_api.dart";
class ClipVectorDB {
static final Logger _logger = Logger("ClipVectorDB");
static const _databaseName = "ente.ml.vectordb.clip";
static final BigInt _embeddingDimension = BigInt.from(512);
static Logger get logger => _logger;
// Singleton pattern
ClipVectorDB._privateConstructor();
static final instance = ClipVectorDB._privateConstructor();
factory ClipVectorDB() => instance;
// only have a single app-wide reference to the database
static Future<VectorDb>? _vectorDbFuture;
Future<VectorDb> get _vectorDB async {
_vectorDbFuture ??= _initVectorDB();
return _vectorDbFuture!;
}
Future<VectorDb> _initVectorDB() async {
final documentsDirectory = await getApplicationDocumentsDirectory();
final String databaseDirectory =
join(documentsDirectory.path, _databaseName);
_logger.info("Opening vectorDB access: DB path " + databaseDirectory);
final vectorDB = VectorDb(
filePath: databaseDirectory,
dimensions: _embeddingDimension,
);
final stats = await getIndexStats(vectorDB);
_logger.info("VectorDB connection opened with stats: ${stats.toString()}");
return vectorDB;
}
Future<void> insertEmbedding({
required int fileID,
required List<double> embedding,
}) async {
final db = await _vectorDB;
try {
await db.addVector(key: BigInt.from(fileID), vector: embedding);
} catch (e, s) {
_logger.severe("Error inserting embedding", e, s);
rethrow;
}
}
Future<void> bulkInsertEmbeddings({
required List<int> fileIDs,
required List<Float32List> embeddings,
}) async {
final db = await _vectorDB;
final bigKeys = Uint64List.fromList(fileIDs);
try {
await db.bulkAddVectors(keys: bigKeys, vectors: embeddings);
} catch (e, s) {
_logger.severe("Error bulk inserting embeddings", e, s);
rethrow;
}
}
Future<List<EmbeddingVector>> getEmbeddings(List<int> fileIDs) async {
final db = await _vectorDB;
try {
final keys = Uint64List.fromList(fileIDs);
final vectors = await db.bulkGetVectors(keys: keys);
return List.generate(
vectors.length,
(index) => EmbeddingVector(
fileID: fileIDs[index],
embedding: vectors[index],
),
);
} catch (e, s) {
_logger.severe("Error getting embeddings", e, s);
rethrow;
}
}
Future<void> deleteEmbeddings(List<int> fileIDs) async {
final db = await _vectorDB;
try {
final deletedCount =
await db.bulkRemoveVectors(keys: Uint64List.fromList(fileIDs));
_logger.info(
"Deleted $deletedCount embeddings, from ${fileIDs.length} keys",
);
} catch (e, s) {
_logger.severe("Error bulk deleting specific embeddings", e, s);
rethrow;
}
}
Future<void> deleteAllEmbeddings() async {
final db = await _vectorDB;
try {
await db.resetIndex();
} catch (e, s) {
_logger.severe("Error deleting all embeddings", e, s);
rethrow;
}
}
Future<void> deleteIndex() async {
final db = await _vectorDB;
try {
await db.deleteIndex();
_vectorDbFuture = null;
} catch (e, s) {
_logger.severe("Error deleting index", e, s);
rethrow;
}
}
Future<VectorDbStats> getIndexStats([VectorDb? db]) async {
db ??= await _vectorDB;
try {
final stats = await db.getIndexStats();
return VectorDbStats(
size: stats.$1.toInt(),
capacity: stats.$2.toInt(),
dimensions: stats.$3.toInt(),
fileSize: stats.$4.toInt(),
memoryUsage: stats.$5.toInt(),
expansionAdd: stats.$6.toInt(),
expansionSearch: stats.$7.toInt(),
);
} catch (e, s) {
_logger.severe("Error getting index stats", e, s);
rethrow;
}
}
Future<(Uint64List, Float32List)> searchClosestVectors(
List<double> query,
int count,
) async {
final db = await _vectorDB;
try {
final result =
await db.searchVectors(query: query, count: BigInt.from(count));
return result;
} catch (e, s) {
_logger.severe("Error searching closest vectors", e, s);
rethrow;
}
}
Future<(BigInt, double)> searchClosestVector(
List<double> query,
) async {
final db = await _vectorDB;
try {
final result = await db.searchVectors(query: query, count: BigInt.one);
return (result.$1[0], result.$2[0]);
} catch (e, s) {
_logger.severe("Error searching closest vector", e, s);
rethrow;
}
}
Future<Map<String, List<QueryResult>>> computeBulkSimilarities(
Map<String, List<double>> textQueryToEmbeddingMap,
Map<String, double> minimumSimilarityMap,
) async {
try {
final queryToResults = <String, List<QueryResult>>{};
for (final MapEntry<String, List<double>> entry
in textQueryToEmbeddingMap.entries) {
final query = entry.key;
final minimumSimilarity = minimumSimilarityMap[query]!;
final textEmbedding = entry.value;
final (potentialFileIDs, distances) =
await searchClosestVectors(textEmbedding, 1000);
final queryResults = <QueryResult>[];
for (var i = 0; i < potentialFileIDs.length; i++) {
final similarity = 1 - distances[i];
if (similarity >= minimumSimilarity) {
queryResults
.add(QueryResult(potentialFileIDs[i].toInt(), similarity));
} else {
break;
}
}
queryToResults[query] = queryResults;
}
return queryToResults;
} catch (e, s) {
_logger.severe(
"Could not bulk find embeddings similarities using vector DB",
e,
s,
);
rethrow;
}
}
}
class VectorDbStats {
final int size;
final int capacity;
final int dimensions;
// in bytes
final int fileSize;
final int memoryUsage;
final int expansionAdd;
final int expansionSearch;
VectorDbStats({
required this.size,
required this.capacity,
required this.dimensions,
required this.fileSize,
required this.memoryUsage,
required this.expansionAdd,
required this.expansionSearch,
});
@override
String toString() {
return "VectorDbStats(size: $size, capacity: $capacity, dimensions: $dimensions, file size on disk (bytes): $fileSize, memory usage (bytes): $memoryUsage, expansionAdd: $expansionAdd, expansionSearch: $expansionSearch)";
}
}

View File

@@ -1,4 +1,5 @@
import 'dart:async';
import "dart:io" show File;
import "dart:math";
import "package:collection/collection.dart";
@@ -9,6 +10,7 @@ import 'package:path_provider/path_provider.dart';
import "package:photos/core/event_bus.dart";
import "package:photos/db/common/base.dart";
import "package:photos/db/ml/base.dart";
import "package:photos/db/ml/clip_vector_db.dart";
import "package:photos/db/ml/db_model_mappers.dart";
import 'package:photos/db/ml/schema.dart';
import "package:photos/events/embedding_updated_event.dart";
@@ -18,6 +20,7 @@ import "package:photos/models/ml/face/face.dart";
import "package:photos/models/ml/face/face_with_embedding.dart";
import "package:photos/models/ml/ml_versions.dart";
import "package:photos/models/ml/vector.dart";
import "package:photos/service_locator.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_db_info_for_clustering.dart";
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
import "package:photos/services/machine_learning/ml_result.dart";
@@ -84,6 +87,8 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
"MLDataDB Migration took ${stopwatch.elapsedMilliseconds} ms",
);
stopwatch.stop();
_logger.info("Starting CLIP vector DB migration check unawaited");
if (flagService.enableVectorDb) unawaited(checkMigrateFillClipVectorDB());
return asyncDBConnection;
}
@@ -1249,6 +1254,121 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
return embeddings;
}
Future<void> checkMigrateFillClipVectorDB({bool force = false}) async {
_logger.info("Waiting for ClipVectorDB to be ready");
await Future.delayed(const Duration(milliseconds: 100));
_logger.info("Checking if ClipVectorDB migration is needed");
// Check if vector DB migration has run
_logger.info("Checking if ClipVectorDB migration has run");
final documentsDirectory = await getApplicationDocumentsDirectory();
final migrationFlagFile =
File(join(documentsDirectory.path, 'clip_vector_migration_done'));
if (await migrationFlagFile.exists() && !force) {
_logger.info("ClipVectorDB migration not needed, already done");
return;
}
// Get total count first to track progress
_logger.info("Getting total count of clip embeddings");
final db = await instance.asyncDB;
final countResult =
await db.getAll('SELECT COUNT($fileIDColumn) as total FROM $clipTable');
final totalCount = countResult.first['total'] as int;
if (totalCount == 0) {
_logger.info("No clip embeddings to migrate");
await migrationFlagFile.create();
return;
}
_logger.info("Total count of clip embeddings: $totalCount");
_logger.info("First time referencing ClipVectorDB in migration");
final clipVectorDB = ClipVectorDB.instance;
_logger.info("ClipVectorDB referenced");
await clipVectorDB.deleteAllEmbeddings();
_logger.info("ClipVectorDB all embeddings cleared");
_logger
.info("Starting migration of $totalCount clip embeddings to vector DB");
const batchSize = 1000;
int offset = 0;
int processedCount = 0;
int weirdCount = 0;
int whileCount = 0;
final stopwatch = Stopwatch()..start();
try {
while (true) {
whileCount++;
_logger.info("$whileCount st round of while loop");
// Allow some time for any GC to finish
await Future.delayed(const Duration(milliseconds: 100));
_logger.info("Reading $batchSize rows from DB");
final List<Map<String, dynamic>> results = await db.getAll('''
SELECT $fileIDColumn, $embeddingColumn
FROM $clipTable
ORDER BY $fileIDColumn DESC
LIMIT $batchSize OFFSET $offset
''');
_logger.info("Got ${results.length} results from DB");
if (results.isEmpty) {
_logger.info("No more results, breaking out of while loop");
break;
}
_logger.info("Processing ${results.length} results");
final List<int> fileIDs = [];
final List<Float32List> embeddings = [];
for (final result in results) {
final embedding =
Float32List.view((result[embeddingColumn] as Uint8List).buffer);
if (embedding.length == 512) {
fileIDs.add(result[fileIDColumn] as int);
embeddings.add(Float32List.view(result[embeddingColumn].buffer));
} else {
weirdCount++;
}
}
_logger.info(
"Got ${fileIDs.length} valid embeddings, $weirdCount weird embeddings",
);
await ClipVectorDB.instance
.bulkInsertEmbeddings(fileIDs: fileIDs, embeddings: embeddings);
_logger.info("Inserted ${fileIDs.length} embeddings to ClipVectorDB");
processedCount += fileIDs.length;
offset += batchSize;
_logger.info(
"migrated $processedCount/$totalCount embeddings to ClipVectorDB",
);
if (processedCount >= totalCount) {
_logger.info("All embeddings migrated, breaking out of while loop");
break;
}
_logger.info("Clearing out embeddings and fileIDs");
embeddings.clear();
fileIDs.clear();
results.clear();
// Allow some time for any GC to finish
_logger.info("Waiting for 100ms for GC to finish");
await Future.delayed(const Duration(milliseconds: 100));
}
_logger.info(
"migrated all $totalCount embeddings to ClipVectorDB in ${stopwatch.elapsed.inMilliseconds} ms, with $weirdCount weird embeddings not migrated",
);
await migrationFlagFile.create();
_logger.info("ClipVectorDB migration done, flag file created");
} catch (e) {
_logger.severe(
"Error migrating ClipVectorDB after ${stopwatch.elapsed.inMilliseconds} ms, clearing out DB again",
e,
);
await clipVectorDB.deleteAllEmbeddings();
rethrow;
} finally {
stopwatch.stop();
}
}
// Get indexed FileIDs
@override
Future<Map<int, int>> clipIndexedFileWithVersion() async {
@@ -1282,12 +1402,25 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
'INSERT OR REPLACE INTO $clipTable ($fileIDColumn, $embeddingColumn, $mlVersionColumn) VALUES (?, ?, ?)',
_getRowFromEmbedding(embeddings.first),
);
if (flagService.enableVectorDb) {
await ClipVectorDB.instance.insertEmbedding(
fileID: embeddings.first.fileID,
embedding: embeddings.first.embedding,
);
}
} else {
final inputs = embeddings.map((e) => _getRowFromEmbedding(e)).toList();
await db.executeBatch(
'INSERT OR REPLACE INTO $clipTable ($fileIDColumn, $embeddingColumn, $mlVersionColumn) values(?, ?, ?)',
inputs,
);
if (flagService.enableVectorDb) {
await ClipVectorDB.instance.bulkInsertEmbeddings(
fileIDs: embeddings.map((e) => e.fileID).toList(),
embeddings:
embeddings.map((e) => Float32List.fromList(e.embedding)).toList(),
);
}
}
Bus.instance.fire(EmbeddingUpdatedEvent());
}
@@ -1298,6 +1431,9 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
await db.execute(
'DELETE FROM $clipTable WHERE $fileIDColumn IN (${fileIDs.join(", ")})',
);
if (flagService.enableVectorDb) {
await ClipVectorDB.instance.deleteEmbeddings(fileIDs);
}
Bus.instance.fire(EmbeddingUpdatedEvent());
}
@@ -1305,6 +1441,9 @@ class MLDataDB with SqlDbBase implements IMLDataDB<int> {
Future<void> deleteClipIndexes() async {
final db = await instance.asyncDB;
await db.execute('DELETE FROM $clipTable');
if (flagService.enableVectorDb) {
await ClipVectorDB.instance.deleteAllEmbeddings();
}
Bus.instance.fire(EmbeddingUpdatedEvent());
}

View File

@@ -37,27 +37,32 @@ class CenterBox extends $pb.GeneratedMessage {
return $result;
}
CenterBox._() : super();
factory CenterBox.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
factory CenterBox.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
factory CenterBox.fromBuffer($core.List<$core.int> i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromBuffer(i, r);
factory CenterBox.fromJson($core.String i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromJson(i, r);
static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'CenterBox', package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.common'), createEmptyInstance: create)
static final $pb.BuilderInfo _i = $pb.BuilderInfo(
_omitMessageNames ? '' : 'CenterBox',
package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.common'),
createEmptyInstance: create)
..a<$core.double>(1, _omitFieldNames ? '' : 'x', $pb.PbFieldType.OF)
..a<$core.double>(2, _omitFieldNames ? '' : 'y', $pb.PbFieldType.OF)
..a<$core.double>(3, _omitFieldNames ? '' : 'height', $pb.PbFieldType.OF)
..a<$core.double>(4, _omitFieldNames ? '' : 'width', $pb.PbFieldType.OF)
..hasRequiredFields = false
;
..hasRequiredFields = false;
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
CenterBox clone() => CenterBox()..mergeFromMessage(this);
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
CenterBox copyWith(void Function(CenterBox) updates) => super.copyWith((message) => updates(message as CenterBox)) as CenterBox;
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
CenterBox copyWith(void Function(CenterBox) updates) =>
super.copyWith((message) => updates(message as CenterBox)) as CenterBox;
$pb.BuilderInfo get info_ => _i;
@@ -66,13 +71,17 @@ class CenterBox extends $pb.GeneratedMessage {
CenterBox createEmptyInstance() => create();
static $pb.PbList<CenterBox> createRepeated() => $pb.PbList<CenterBox>();
@$core.pragma('dart2js:noInline')
static CenterBox getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<CenterBox>(create);
static CenterBox getDefault() =>
_defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<CenterBox>(create);
static CenterBox? _defaultInstance;
@$pb.TagNumber(1)
$core.double get x => $_getN(0);
@$pb.TagNumber(1)
set x($core.double v) { $_setFloat(0, v); }
set x($core.double v) {
$_setFloat(0, v);
}
@$pb.TagNumber(1)
$core.bool hasX() => $_has(0);
@$pb.TagNumber(1)
@@ -81,7 +90,10 @@ class CenterBox extends $pb.GeneratedMessage {
@$pb.TagNumber(2)
$core.double get y => $_getN(1);
@$pb.TagNumber(2)
set y($core.double v) { $_setFloat(1, v); }
set y($core.double v) {
$_setFloat(1, v);
}
@$pb.TagNumber(2)
$core.bool hasY() => $_has(1);
@$pb.TagNumber(2)
@@ -90,7 +102,10 @@ class CenterBox extends $pb.GeneratedMessage {
@$pb.TagNumber(3)
$core.double get height => $_getN(2);
@$pb.TagNumber(3)
set height($core.double v) { $_setFloat(2, v); }
set height($core.double v) {
$_setFloat(2, v);
}
@$pb.TagNumber(3)
$core.bool hasHeight() => $_has(2);
@$pb.TagNumber(3)
@@ -99,13 +114,16 @@ class CenterBox extends $pb.GeneratedMessage {
@$pb.TagNumber(4)
$core.double get width => $_getN(3);
@$pb.TagNumber(4)
set width($core.double v) { $_setFloat(3, v); }
set width($core.double v) {
$_setFloat(3, v);
}
@$pb.TagNumber(4)
$core.bool hasWidth() => $_has(3);
@$pb.TagNumber(4)
void clearWidth() => clearField(4);
}
const _omitFieldNames = $core.bool.fromEnvironment('protobuf.omit_field_names');
const _omitMessageNames = $core.bool.fromEnvironment('protobuf.omit_message_names');
const _omitMessageNames =
$core.bool.fromEnvironment('protobuf.omit_message_names');

View File

@@ -8,4 +8,3 @@
// ignore_for_file: constant_identifier_names, library_prefixes
// ignore_for_file: non_constant_identifier_names, prefer_final_fields
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import

View File

@@ -35,4 +35,3 @@ final $typed_data.Uint8List centerBoxDescriptor = $convert.base64Decode(
'CglDZW50ZXJCb3gSEQoBeBgBIAEoAkgAUgF4iAEBEhEKAXkYAiABKAJIAVIBeYgBARIbCgZoZW'
'lnaHQYAyABKAJIAlIGaGVpZ2h0iAEBEhkKBXdpZHRoGAQgASgCSANSBXdpZHRoiAEBQgQKAl94'
'QgQKAl95QgkKB19oZWlnaHRCCAoGX3dpZHRo');

View File

@@ -11,4 +11,3 @@
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import
export 'box.pb.dart';

View File

@@ -29,25 +29,30 @@ class EPoint extends $pb.GeneratedMessage {
return $result;
}
EPoint._() : super();
factory EPoint.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
factory EPoint.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
factory EPoint.fromBuffer($core.List<$core.int> i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromBuffer(i, r);
factory EPoint.fromJson($core.String i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromJson(i, r);
static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'EPoint', package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.common'), createEmptyInstance: create)
static final $pb.BuilderInfo _i = $pb.BuilderInfo(
_omitMessageNames ? '' : 'EPoint',
package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.common'),
createEmptyInstance: create)
..a<$core.double>(1, _omitFieldNames ? '' : 'x', $pb.PbFieldType.OF)
..a<$core.double>(2, _omitFieldNames ? '' : 'y', $pb.PbFieldType.OF)
..hasRequiredFields = false
;
..hasRequiredFields = false;
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
EPoint clone() => EPoint()..mergeFromMessage(this);
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
EPoint copyWith(void Function(EPoint) updates) => super.copyWith((message) => updates(message as EPoint)) as EPoint;
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
EPoint copyWith(void Function(EPoint) updates) =>
super.copyWith((message) => updates(message as EPoint)) as EPoint;
$pb.BuilderInfo get info_ => _i;
@@ -56,13 +61,17 @@ class EPoint extends $pb.GeneratedMessage {
EPoint createEmptyInstance() => create();
static $pb.PbList<EPoint> createRepeated() => $pb.PbList<EPoint>();
@$core.pragma('dart2js:noInline')
static EPoint getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<EPoint>(create);
static EPoint getDefault() =>
_defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<EPoint>(create);
static EPoint? _defaultInstance;
@$pb.TagNumber(1)
$core.double get x => $_getN(0);
@$pb.TagNumber(1)
set x($core.double v) { $_setFloat(0, v); }
set x($core.double v) {
$_setFloat(0, v);
}
@$pb.TagNumber(1)
$core.bool hasX() => $_has(0);
@$pb.TagNumber(1)
@@ -71,13 +80,16 @@ class EPoint extends $pb.GeneratedMessage {
@$pb.TagNumber(2)
$core.double get y => $_getN(1);
@$pb.TagNumber(2)
set y($core.double v) { $_setFloat(1, v); }
set y($core.double v) {
$_setFloat(1, v);
}
@$pb.TagNumber(2)
$core.bool hasY() => $_has(1);
@$pb.TagNumber(2)
void clearY() => clearField(2);
}
const _omitFieldNames = $core.bool.fromEnvironment('protobuf.omit_field_names');
const _omitMessageNames = $core.bool.fromEnvironment('protobuf.omit_message_names');
const _omitMessageNames =
$core.bool.fromEnvironment('protobuf.omit_message_names');

View File

@@ -8,4 +8,3 @@
// ignore_for_file: constant_identifier_names, library_prefixes
// ignore_for_file: non_constant_identifier_names, prefer_final_fields
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import

View File

@@ -30,4 +30,3 @@ const EPoint$json = {
final $typed_data.Uint8List ePointDescriptor = $convert.base64Decode(
'CgZFUG9pbnQSEQoBeBgBIAEoAkgAUgF4iAEBEhEKAXkYAiABKAJIAVIBeYgBAUIECgJfeEIECg'
'JfeQ==');

View File

@@ -11,4 +11,3 @@
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import
export 'point.pb.dart';

View File

@@ -26,24 +26,29 @@ class EVector extends $pb.GeneratedMessage {
return $result;
}
EVector._() : super();
factory EVector.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
factory EVector.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
factory EVector.fromBuffer($core.List<$core.int> i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromBuffer(i, r);
factory EVector.fromJson($core.String i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromJson(i, r);
static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'EVector', package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.common'), createEmptyInstance: create)
static final $pb.BuilderInfo _i = $pb.BuilderInfo(
_omitMessageNames ? '' : 'EVector',
package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.common'),
createEmptyInstance: create)
..p<$core.double>(1, _omitFieldNames ? '' : 'values', $pb.PbFieldType.KD)
..hasRequiredFields = false
;
..hasRequiredFields = false;
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
EVector clone() => EVector()..mergeFromMessage(this);
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
EVector copyWith(void Function(EVector) updates) => super.copyWith((message) => updates(message as EVector)) as EVector;
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
EVector copyWith(void Function(EVector) updates) =>
super.copyWith((message) => updates(message as EVector)) as EVector;
$pb.BuilderInfo get info_ => _i;
@@ -52,13 +57,14 @@ class EVector extends $pb.GeneratedMessage {
EVector createEmptyInstance() => create();
static $pb.PbList<EVector> createRepeated() => $pb.PbList<EVector>();
@$core.pragma('dart2js:noInline')
static EVector getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<EVector>(create);
static EVector getDefault() =>
_defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<EVector>(create);
static EVector? _defaultInstance;
@$pb.TagNumber(1)
$core.List<$core.double> get values => $_getList(0);
}
const _omitFieldNames = $core.bool.fromEnvironment('protobuf.omit_field_names');
const _omitMessageNames = $core.bool.fromEnvironment('protobuf.omit_message_names');
const _omitMessageNames =
$core.bool.fromEnvironment('protobuf.omit_message_names');

View File

@@ -8,4 +8,3 @@
// ignore_for_file: constant_identifier_names, library_prefixes
// ignore_for_file: non_constant_identifier_names, prefer_final_fields
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import

View File

@@ -22,6 +22,5 @@ const EVector$json = {
};
/// Descriptor for `EVector`. Decode as a `google.protobuf.DescriptorProto`.
final $typed_data.Uint8List eVectorDescriptor = $convert.base64Decode(
'CgdFVmVjdG9yEhYKBnZhbHVlcxgBIAMoAVIGdmFsdWVz');
final $typed_data.Uint8List eVectorDescriptor =
$convert.base64Decode('CgdFVmVjdG9yEhYKBnZhbHVlcxgBIAMoAVIGdmFsdWVz');

View File

@@ -11,4 +11,3 @@
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import
export 'vector.pb.dart';

View File

@@ -31,25 +31,32 @@ class Detection extends $pb.GeneratedMessage {
return $result;
}
Detection._() : super();
factory Detection.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
factory Detection.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
factory Detection.fromBuffer($core.List<$core.int> i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromBuffer(i, r);
factory Detection.fromJson($core.String i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromJson(i, r);
static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'Detection', package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.ml'), createEmptyInstance: create)
..aOM<$0.CenterBox>(1, _omitFieldNames ? '' : 'box', subBuilder: $0.CenterBox.create)
..aOM<$1.EPoint>(2, _omitFieldNames ? '' : 'landmarks', subBuilder: $1.EPoint.create)
..hasRequiredFields = false
;
static final $pb.BuilderInfo _i = $pb.BuilderInfo(
_omitMessageNames ? '' : 'Detection',
package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.ml'),
createEmptyInstance: create)
..aOM<$0.CenterBox>(1, _omitFieldNames ? '' : 'box',
subBuilder: $0.CenterBox.create)
..aOM<$1.EPoint>(2, _omitFieldNames ? '' : 'landmarks',
subBuilder: $1.EPoint.create)
..hasRequiredFields = false;
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
Detection clone() => Detection()..mergeFromMessage(this);
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
Detection copyWith(void Function(Detection) updates) => super.copyWith((message) => updates(message as Detection)) as Detection;
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
Detection copyWith(void Function(Detection) updates) =>
super.copyWith((message) => updates(message as Detection)) as Detection;
$pb.BuilderInfo get info_ => _i;
@@ -58,13 +65,17 @@ class Detection extends $pb.GeneratedMessage {
Detection createEmptyInstance() => create();
static $pb.PbList<Detection> createRepeated() => $pb.PbList<Detection>();
@$core.pragma('dart2js:noInline')
static Detection getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<Detection>(create);
static Detection getDefault() =>
_defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<Detection>(create);
static Detection? _defaultInstance;
@$pb.TagNumber(1)
$0.CenterBox get box => $_getN(0);
@$pb.TagNumber(1)
set box($0.CenterBox v) { setField(1, v); }
set box($0.CenterBox v) {
setField(1, v);
}
@$pb.TagNumber(1)
$core.bool hasBox() => $_has(0);
@$pb.TagNumber(1)
@@ -75,7 +86,10 @@ class Detection extends $pb.GeneratedMessage {
@$pb.TagNumber(2)
$1.EPoint get landmarks => $_getN(1);
@$pb.TagNumber(2)
set landmarks($1.EPoint v) { setField(2, v); }
set landmarks($1.EPoint v) {
setField(2, v);
}
@$pb.TagNumber(2)
$core.bool hasLandmarks() => $_has(1);
@$pb.TagNumber(2)
@@ -103,26 +117,33 @@ class Face extends $pb.GeneratedMessage {
return $result;
}
Face._() : super();
factory Face.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
factory Face.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
factory Face.fromBuffer($core.List<$core.int> i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromBuffer(i, r);
factory Face.fromJson($core.String i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromJson(i, r);
static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'Face', package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.ml'), createEmptyInstance: create)
static final $pb.BuilderInfo _i = $pb.BuilderInfo(
_omitMessageNames ? '' : 'Face',
package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.ml'),
createEmptyInstance: create)
..aOS(1, _omitFieldNames ? '' : 'id')
..aOM<Detection>(2, _omitFieldNames ? '' : 'detection', subBuilder: Detection.create)
..a<$core.double>(3, _omitFieldNames ? '' : 'confidence', $pb.PbFieldType.OF)
..hasRequiredFields = false
;
..aOM<Detection>(2, _omitFieldNames ? '' : 'detection',
subBuilder: Detection.create)
..a<$core.double>(
3, _omitFieldNames ? '' : 'confidence', $pb.PbFieldType.OF)
..hasRequiredFields = false;
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
Face clone() => Face()..mergeFromMessage(this);
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
Face copyWith(void Function(Face) updates) => super.copyWith((message) => updates(message as Face)) as Face;
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
Face copyWith(void Function(Face) updates) =>
super.copyWith((message) => updates(message as Face)) as Face;
$pb.BuilderInfo get info_ => _i;
@@ -131,13 +152,17 @@ class Face extends $pb.GeneratedMessage {
Face createEmptyInstance() => create();
static $pb.PbList<Face> createRepeated() => $pb.PbList<Face>();
@$core.pragma('dart2js:noInline')
static Face getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<Face>(create);
static Face getDefault() =>
_defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<Face>(create);
static Face? _defaultInstance;
@$pb.TagNumber(1)
$core.String get id => $_getSZ(0);
@$pb.TagNumber(1)
set id($core.String v) { $_setString(0, v); }
set id($core.String v) {
$_setString(0, v);
}
@$pb.TagNumber(1)
$core.bool hasId() => $_has(0);
@$pb.TagNumber(1)
@@ -146,7 +171,10 @@ class Face extends $pb.GeneratedMessage {
@$pb.TagNumber(2)
Detection get detection => $_getN(1);
@$pb.TagNumber(2)
set detection(Detection v) { setField(2, v); }
set detection(Detection v) {
setField(2, v);
}
@$pb.TagNumber(2)
$core.bool hasDetection() => $_has(1);
@$pb.TagNumber(2)
@@ -157,13 +185,16 @@ class Face extends $pb.GeneratedMessage {
@$pb.TagNumber(3)
$core.double get confidence => $_getN(2);
@$pb.TagNumber(3)
set confidence($core.double v) { $_setFloat(2, v); }
set confidence($core.double v) {
$_setFloat(2, v);
}
@$pb.TagNumber(3)
$core.bool hasConfidence() => $_has(2);
@$pb.TagNumber(3)
void clearConfidence() => clearField(3);
}
const _omitFieldNames = $core.bool.fromEnvironment('protobuf.omit_field_names');
const _omitMessageNames = $core.bool.fromEnvironment('protobuf.omit_message_names');
const _omitMessageNames =
$core.bool.fromEnvironment('protobuf.omit_message_names');

View File

@@ -8,4 +8,3 @@
// ignore_for_file: constant_identifier_names, library_prefixes
// ignore_for_file: non_constant_identifier_names, prefer_final_fields
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import

View File

@@ -17,8 +17,26 @@ import 'dart:typed_data' as $typed_data;
const Detection$json = {
'1': 'Detection',
'2': [
{'1': 'box', '3': 1, '4': 1, '5': 11, '6': '.ente.common.CenterBox', '9': 0, '10': 'box', '17': true},
{'1': 'landmarks', '3': 2, '4': 1, '5': 11, '6': '.ente.common.EPoint', '9': 1, '10': 'landmarks', '17': true},
{
'1': 'box',
'3': 1,
'4': 1,
'5': 11,
'6': '.ente.common.CenterBox',
'9': 0,
'10': 'box',
'17': true
},
{
'1': 'landmarks',
'3': 2,
'4': 1,
'5': 11,
'6': '.ente.common.EPoint',
'9': 1,
'10': 'landmarks',
'17': true
},
],
'8': [
{'1': '_box'},
@@ -37,8 +55,25 @@ const Face$json = {
'1': 'Face',
'2': [
{'1': 'id', '3': 1, '4': 1, '5': 9, '9': 0, '10': 'id', '17': true},
{'1': 'detection', '3': 2, '4': 1, '5': 11, '6': '.ente.ml.Detection', '9': 1, '10': 'detection', '17': true},
{'1': 'confidence', '3': 3, '4': 1, '5': 2, '9': 2, '10': 'confidence', '17': true},
{
'1': 'detection',
'3': 2,
'4': 1,
'5': 11,
'6': '.ente.ml.Detection',
'9': 1,
'10': 'detection',
'17': true
},
{
'1': 'confidence',
'3': 3,
'4': 1,
'5': 2,
'9': 2,
'10': 'confidence',
'17': true
},
],
'8': [
{'1': '_id'},
@@ -52,4 +87,3 @@ final $typed_data.Uint8List faceDescriptor = $convert.base64Decode(
'CgRGYWNlEhMKAmlkGAEgASgJSABSAmlkiAEBEjUKCWRldGVjdGlvbhgCIAEoCzISLmVudGUubW'
'wuRGV0ZWN0aW9uSAFSCWRldGVjdGlvbogBARIjCgpjb25maWRlbmNlGAMgASgCSAJSCmNvbmZp'
'ZGVuY2WIAQFCBQoDX2lkQgwKCl9kZXRlY3Rpb25CDQoLX2NvbmZpZGVuY2U=');

View File

@@ -11,4 +11,3 @@
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import
export 'face.pb.dart';

View File

@@ -31,25 +31,30 @@ class FileML extends $pb.GeneratedMessage {
return $result;
}
FileML._() : super();
factory FileML.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
factory FileML.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
factory FileML.fromBuffer($core.List<$core.int> i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromBuffer(i, r);
factory FileML.fromJson($core.String i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromJson(i, r);
static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'FileML', package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.ml'), createEmptyInstance: create)
static final $pb.BuilderInfo _i = $pb.BuilderInfo(
_omitMessageNames ? '' : 'FileML',
package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.ml'),
createEmptyInstance: create)
..aInt64(1, _omitFieldNames ? '' : 'id')
..p<$core.double>(2, _omitFieldNames ? '' : 'clip', $pb.PbFieldType.KD)
..hasRequiredFields = false
;
..hasRequiredFields = false;
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
FileML clone() => FileML()..mergeFromMessage(this);
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
FileML copyWith(void Function(FileML) updates) => super.copyWith((message) => updates(message as FileML)) as FileML;
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
FileML copyWith(void Function(FileML) updates) =>
super.copyWith((message) => updates(message as FileML)) as FileML;
$pb.BuilderInfo get info_ => _i;
@@ -58,13 +63,17 @@ class FileML extends $pb.GeneratedMessage {
FileML createEmptyInstance() => create();
static $pb.PbList<FileML> createRepeated() => $pb.PbList<FileML>();
@$core.pragma('dart2js:noInline')
static FileML getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<FileML>(create);
static FileML getDefault() =>
_defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<FileML>(create);
static FileML? _defaultInstance;
@$pb.TagNumber(1)
$fixnum.Int64 get id => $_getI64(0);
@$pb.TagNumber(1)
set id($fixnum.Int64 v) { $_setInt64(0, v); }
set id($fixnum.Int64 v) {
$_setInt64(0, v);
}
@$pb.TagNumber(1)
$core.bool hasId() => $_has(0);
@$pb.TagNumber(1)
@@ -101,28 +110,34 @@ class FileFaces extends $pb.GeneratedMessage {
return $result;
}
FileFaces._() : super();
factory FileFaces.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r);
factory FileFaces.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r);
factory FileFaces.fromBuffer($core.List<$core.int> i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromBuffer(i, r);
factory FileFaces.fromJson($core.String i,
[$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) =>
create()..mergeFromJson(i, r);
static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'FileFaces', package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.ml'), createEmptyInstance: create)
..pc<$2.Face>(1, _omitFieldNames ? '' : 'faces', $pb.PbFieldType.PM, subBuilder: $2.Face.create)
static final $pb.BuilderInfo _i = $pb.BuilderInfo(
_omitMessageNames ? '' : 'FileFaces',
package: const $pb.PackageName(_omitMessageNames ? '' : 'ente.ml'),
createEmptyInstance: create)
..pc<$2.Face>(1, _omitFieldNames ? '' : 'faces', $pb.PbFieldType.PM,
subBuilder: $2.Face.create)
..a<$core.int>(2, _omitFieldNames ? '' : 'height', $pb.PbFieldType.O3)
..a<$core.int>(3, _omitFieldNames ? '' : 'width', $pb.PbFieldType.O3)
..a<$core.int>(4, _omitFieldNames ? '' : 'version', $pb.PbFieldType.O3)
..aOS(5, _omitFieldNames ? '' : 'error')
..hasRequiredFields = false
;
..hasRequiredFields = false;
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.deepCopy] instead. '
'Will be removed in next major version')
FileFaces clone() => FileFaces()..mergeFromMessage(this);
@$core.Deprecated(
'Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
FileFaces copyWith(void Function(FileFaces) updates) => super.copyWith((message) => updates(message as FileFaces)) as FileFaces;
@$core.Deprecated('Using this can add significant overhead to your binary. '
'Use [GeneratedMessageGenericExtensions.rebuild] instead. '
'Will be removed in next major version')
FileFaces copyWith(void Function(FileFaces) updates) =>
super.copyWith((message) => updates(message as FileFaces)) as FileFaces;
$pb.BuilderInfo get info_ => _i;
@@ -131,7 +146,8 @@ class FileFaces extends $pb.GeneratedMessage {
FileFaces createEmptyInstance() => create();
static $pb.PbList<FileFaces> createRepeated() => $pb.PbList<FileFaces>();
@$core.pragma('dart2js:noInline')
static FileFaces getDefault() => _defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<FileFaces>(create);
static FileFaces getDefault() =>
_defaultInstance ??= $pb.GeneratedMessage.$_defaultFor<FileFaces>(create);
static FileFaces? _defaultInstance;
@$pb.TagNumber(1)
@@ -140,7 +156,10 @@ class FileFaces extends $pb.GeneratedMessage {
@$pb.TagNumber(2)
$core.int get height => $_getIZ(1);
@$pb.TagNumber(2)
set height($core.int v) { $_setSignedInt32(1, v); }
set height($core.int v) {
$_setSignedInt32(1, v);
}
@$pb.TagNumber(2)
$core.bool hasHeight() => $_has(1);
@$pb.TagNumber(2)
@@ -149,7 +168,10 @@ class FileFaces extends $pb.GeneratedMessage {
@$pb.TagNumber(3)
$core.int get width => $_getIZ(2);
@$pb.TagNumber(3)
set width($core.int v) { $_setSignedInt32(2, v); }
set width($core.int v) {
$_setSignedInt32(2, v);
}
@$pb.TagNumber(3)
$core.bool hasWidth() => $_has(2);
@$pb.TagNumber(3)
@@ -158,7 +180,10 @@ class FileFaces extends $pb.GeneratedMessage {
@$pb.TagNumber(4)
$core.int get version => $_getIZ(3);
@$pb.TagNumber(4)
set version($core.int v) { $_setSignedInt32(3, v); }
set version($core.int v) {
$_setSignedInt32(3, v);
}
@$pb.TagNumber(4)
$core.bool hasVersion() => $_has(3);
@$pb.TagNumber(4)
@@ -167,13 +192,16 @@ class FileFaces extends $pb.GeneratedMessage {
@$pb.TagNumber(5)
$core.String get error => $_getSZ(4);
@$pb.TagNumber(5)
set error($core.String v) { $_setString(4, v); }
set error($core.String v) {
$_setString(4, v);
}
@$pb.TagNumber(5)
$core.bool hasError() => $_has(4);
@$pb.TagNumber(5)
void clearError() => clearField(5);
}
const _omitFieldNames = $core.bool.fromEnvironment('protobuf.omit_field_names');
const _omitMessageNames = $core.bool.fromEnvironment('protobuf.omit_message_names');
const _omitMessageNames =
$core.bool.fromEnvironment('protobuf.omit_message_names');

View File

@@ -8,4 +8,3 @@
// ignore_for_file: constant_identifier_names, library_prefixes
// ignore_for_file: non_constant_identifier_names, prefer_final_fields
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import

View File

@@ -34,10 +34,25 @@ final $typed_data.Uint8List fileMLDescriptor = $convert.base64Decode(
const FileFaces$json = {
'1': 'FileFaces',
'2': [
{'1': 'faces', '3': 1, '4': 3, '5': 11, '6': '.ente.ml.Face', '10': 'faces'},
{
'1': 'faces',
'3': 1,
'4': 3,
'5': 11,
'6': '.ente.ml.Face',
'10': 'faces'
},
{'1': 'height', '3': 2, '4': 1, '5': 5, '9': 0, '10': 'height', '17': true},
{'1': 'width', '3': 3, '4': 1, '5': 5, '9': 1, '10': 'width', '17': true},
{'1': 'version', '3': 4, '4': 1, '5': 5, '9': 2, '10': 'version', '17': true},
{
'1': 'version',
'3': 4,
'4': 1,
'5': 5,
'9': 2,
'10': 'version',
'17': true
},
{'1': 'error', '3': 5, '4': 1, '5': 9, '9': 3, '10': 'error', '17': true},
],
'8': [
@@ -54,4 +69,3 @@ final $typed_data.Uint8List fileFacesDescriptor = $convert.base64Decode(
'dodBgCIAEoBUgAUgZoZWlnaHSIAQESGQoFd2lkdGgYAyABKAVIAVIFd2lkdGiIAQESHQoHdmVy'
'c2lvbhgEIAEoBUgCUgd2ZXJzaW9uiAEBEhkKBWVycm9yGAUgASgJSANSBWVycm9yiAEBQgkKB1'
'9oZWlnaHRCCAoGX3dpZHRoQgoKCF92ZXJzaW9uQggKBl9lcnJvcg==');

View File

@@ -11,4 +11,3 @@
// ignore_for_file: unnecessary_import, unnecessary_this, unused_import
export 'fileml.pb.dart';

View File

@@ -43,6 +43,7 @@ import 'package:photos/services/sync/remote_sync_service.dart';
import "package:photos/services/sync/sync_service.dart";
import "package:photos/services/video_preview_service.dart";
import "package:photos/services/wake_lock_service.dart";
import "package:photos/src/rust/frb_generated.dart";
import 'package:photos/ui/tools/app_lock.dart';
import 'package:photos/ui/tools/lock_screen.dart';
import "package:photos/utils/email_util.dart";
@@ -63,6 +64,7 @@ const kFGTaskDeathTimeoutInMicroseconds = 5000000;
void main() async {
debugRepaintRainbowEnabled = false;
await RustLib.init();
WidgetsFlutterBinding.ensureInitialized();
MediaKit.ensureInitialized();

View File

@@ -5,7 +5,6 @@ class SetupSRPRequest {
final String srpA;
final bool isUpdate;
SetupSRPRequest({
required this.srpUserID,
required this.srpSalt,
@@ -82,6 +81,7 @@ class CompleteSRPSetupRequest {
);
}
}
class SrpAttributes {
final String srpUserID;
final String srpSalt;

View File

@@ -507,7 +507,8 @@ ClusteringResult _runCompleteClustering(Map args) {
EVector.fromBuffer(entry.value).values,
dtype: DType.float32,
),
fileCreationTime: fileIDToCreationTime?[getFileIdFromFaceId<int>(entry.key)],
fileCreationTime:
fileIDToCreationTime?[getFileIdFromFaceId<int>(entry.key)],
),
);
}

View File

@@ -1 +1 @@
const imageEmbeddingsKey = "imageEmbeddings";
const imageEmbeddingsKey = "imageEmbeddings";

View File

@@ -1,4 +1,3 @@
class GeneralFaceMlException implements Exception {
final String message;
@@ -26,4 +25,4 @@ class CouldNotRunFaceDetector implements Exception {}
class CouldNotWarpAffine implements Exception {}
class CouldNotRunFaceEmbeddor implements Exception {}
class CouldNotRunFaceEmbeddor implements Exception {}

View File

@@ -3,4 +3,4 @@ class QueryResult {
final double score;
QueryResult(this.id, this.score);
}
}

View File

@@ -8,6 +8,7 @@ import "package:logging/logging.dart";
import "package:photos/core/cache/lru_map.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/files_db.dart";
import "package:photos/db/ml/clip_vector_db.dart";
import "package:photos/db/ml/db.dart";
import 'package:photos/events/embedding_updated_event.dart';
import "package:photos/models/file/file.dart";
@@ -265,10 +266,25 @@ class SemanticSearchService {
required Map<String, double> minimumSimilarityMap,
}) async {
final startTime = DateTime.now();
await _cacheClipVectors();
final Map<String, List<QueryResult>> queryResults = await MLComputer
.instance
.computeBulkSimilarities(textQueryToEmbeddingMap, minimumSimilarityMap);
if (kDebugMode) {
for (final queryText in textQueryToEmbeddingMap.keys) {
final embedding = textQueryToEmbeddingMap[queryText]!;
dev.log("CLIPTEXT Query: $queryText, embedding: $embedding");
}
}
late final Map<String, List<QueryResult>> queryResults;
if (flagService.enableVectorDb) {
queryResults = await ClipVectorDB.instance.computeBulkSimilarities(
textQueryToEmbeddingMap,
minimumSimilarityMap,
);
} else {
await _cacheClipVectors();
queryResults = await MLComputer.instance.computeBulkSimilarities(
textQueryToEmbeddingMap,
minimumSimilarityMap,
);
}
final endTime = DateTime.now();
_logger.info(
"computingSimilarities took for ${textQueryToEmbeddingMap.length} queries " +

View File

@@ -0,0 +1,12 @@
// This file is automatically generated, so please do not edit it.
// @generated by `flutter_rust_bridge`@ 2.11.1.
// ignore_for_file: require_trailing_commas
// ignore_for_file: invalid_use_of_internal_member, unused_import, unnecessary_import
import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart';
import 'package:photos/src/rust/frb_generated.dart';
String greet({required String name}) =>
RustLib.instance.api.crateApiSimpleGreet(name: name);

View File

@@ -0,0 +1,48 @@
// This file is automatically generated, so please do not edit it.
// @generated by `flutter_rust_bridge`@ 2.11.1.
// ignore_for_file: require_trailing_commas
// ignore_for_file: invalid_use_of_internal_member, unused_import, unnecessary_import
import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart';
import 'package:photos/src/rust/frb_generated.dart';
// These functions are ignored because they are not marked as `pub`: `ensure_capacity`, `save_index`
// Rust type: RustOpaqueMoi<flutter_rust_bridge::for_generated::RustAutoOpaqueInner<VectorDB>>
abstract class VectorDb implements RustOpaqueInterface {
Future<void> addVector({required BigInt key, required List<double> vector});
Future<void> bulkAddVectors(
{required Uint64List keys, required List<Float32List> vectors});
Future<List<Float32List>> bulkGetVectors({required Uint64List keys});
Future<BigInt> bulkRemoveVectors({required Uint64List keys});
Future<(List<Uint64List>, List<Float32List>)> bulkSearchVectors(
{required List<Float32List> queries, required BigInt count});
/// Check if a vector with the given key exists in the index.
/// `true` if the index contains the vector with the given key, `false` otherwise.
Future<bool> containsVector({required BigInt key});
Future<void> deleteIndex();
Future<(BigInt, BigInt, BigInt, BigInt, BigInt, BigInt, BigInt)>
getIndexStats();
Future<Float32List> getVector({required BigInt key});
factory VectorDb({required String filePath, required BigInt dimensions}) =>
RustLib.instance.api.crateApiUsearchApiVectorDbNew(
filePath: filePath, dimensions: dimensions);
Future<BigInt> removeVector({required BigInt key});
Future<void> resetIndex();
Future<(Uint64List, Float32List)> searchVectors(
{required List<double> query, required BigInt count});
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,293 @@
// This file is automatically generated, so please do not edit it.
// @generated by `flutter_rust_bridge`@ 2.11.1.
// ignore_for_file: require_trailing_commas
// ignore_for_file: unused_import, unused_element, unnecessary_import, duplicate_ignore, invalid_use_of_internal_member, annotate_overrides, non_constant_identifier_names, curly_braces_in_flow_control_structures, prefer_const_literals_to_create_immutables, unused_field
import 'dart:async';
import 'dart:convert';
import 'dart:ffi' as ffi;
import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated_io.dart';
import 'package:photos/src/rust/api/simple.dart';
import 'package:photos/src/rust/api/usearch_api.dart';
import 'package:photos/src/rust/frb_generated.dart';
abstract class RustLibApiImplPlatform extends BaseApiImpl<RustLibWire> {
RustLibApiImplPlatform({
required super.handler,
required super.wire,
required super.generalizedFrbRustBinding,
required super.portManager,
});
CrossPlatformFinalizerArg get rust_arc_decrement_strong_count_VectorDbPtr => wire
._rust_arc_decrement_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDBPtr;
@protected
VectorDb
dco_decode_Auto_Owned_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
dynamic raw);
@protected
VectorDb
dco_decode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
dynamic raw);
@protected
VectorDb
dco_decode_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
dynamic raw);
@protected
String dco_decode_String(dynamic raw);
@protected
bool dco_decode_bool(dynamic raw);
@protected
double dco_decode_f_32(dynamic raw);
@protected
List<Float32List> dco_decode_list_list_prim_f_32_strict(dynamic raw);
@protected
List<Uint64List> dco_decode_list_list_prim_u_64_strict(dynamic raw);
@protected
List<double> dco_decode_list_prim_f_32_loose(dynamic raw);
@protected
Float32List dco_decode_list_prim_f_32_strict(dynamic raw);
@protected
Uint64List dco_decode_list_prim_u_64_strict(dynamic raw);
@protected
Uint8List dco_decode_list_prim_u_8_strict(dynamic raw);
@protected
(List<Uint64List>, List<Float32List>)
dco_decode_record_list_list_prim_u_64_strict_list_list_prim_f_32_strict(
dynamic raw);
@protected
(
Uint64List,
Float32List
) dco_decode_record_list_prim_u_64_strict_list_prim_f_32_strict(dynamic raw);
@protected
(BigInt, BigInt, BigInt, BigInt, BigInt, BigInt, BigInt)
dco_decode_record_usize_usize_usize_usize_usize_usize_usize(dynamic raw);
@protected
BigInt dco_decode_u_64(dynamic raw);
@protected
int dco_decode_u_8(dynamic raw);
@protected
void dco_decode_unit(dynamic raw);
@protected
BigInt dco_decode_usize(dynamic raw);
@protected
VectorDb
sse_decode_Auto_Owned_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
SseDeserializer deserializer);
@protected
VectorDb
sse_decode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
SseDeserializer deserializer);
@protected
VectorDb
sse_decode_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
SseDeserializer deserializer);
@protected
String sse_decode_String(SseDeserializer deserializer);
@protected
bool sse_decode_bool(SseDeserializer deserializer);
@protected
double sse_decode_f_32(SseDeserializer deserializer);
@protected
List<Float32List> sse_decode_list_list_prim_f_32_strict(
SseDeserializer deserializer);
@protected
List<Uint64List> sse_decode_list_list_prim_u_64_strict(
SseDeserializer deserializer);
@protected
List<double> sse_decode_list_prim_f_32_loose(SseDeserializer deserializer);
@protected
Float32List sse_decode_list_prim_f_32_strict(SseDeserializer deserializer);
@protected
Uint64List sse_decode_list_prim_u_64_strict(SseDeserializer deserializer);
@protected
Uint8List sse_decode_list_prim_u_8_strict(SseDeserializer deserializer);
@protected
(List<Uint64List>, List<Float32List>)
sse_decode_record_list_list_prim_u_64_strict_list_list_prim_f_32_strict(
SseDeserializer deserializer);
@protected
(Uint64List, Float32List)
sse_decode_record_list_prim_u_64_strict_list_prim_f_32_strict(
SseDeserializer deserializer);
@protected
(BigInt, BigInt, BigInt, BigInt, BigInt, BigInt, BigInt)
sse_decode_record_usize_usize_usize_usize_usize_usize_usize(
SseDeserializer deserializer);
@protected
BigInt sse_decode_u_64(SseDeserializer deserializer);
@protected
int sse_decode_u_8(SseDeserializer deserializer);
@protected
void sse_decode_unit(SseDeserializer deserializer);
@protected
BigInt sse_decode_usize(SseDeserializer deserializer);
@protected
int sse_decode_i_32(SseDeserializer deserializer);
@protected
void
sse_encode_Auto_Owned_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
VectorDb self, SseSerializer serializer);
@protected
void
sse_encode_Auto_Ref_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
VectorDb self, SseSerializer serializer);
@protected
void
sse_encode_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
VectorDb self, SseSerializer serializer);
@protected
void sse_encode_String(String self, SseSerializer serializer);
@protected
void sse_encode_bool(bool self, SseSerializer serializer);
@protected
void sse_encode_f_32(double self, SseSerializer serializer);
@protected
void sse_encode_list_list_prim_f_32_strict(
List<Float32List> self, SseSerializer serializer);
@protected
void sse_encode_list_list_prim_u_64_strict(
List<Uint64List> self, SseSerializer serializer);
@protected
void sse_encode_list_prim_f_32_loose(
List<double> self, SseSerializer serializer);
@protected
void sse_encode_list_prim_f_32_strict(
Float32List self, SseSerializer serializer);
@protected
void sse_encode_list_prim_u_64_strict(
Uint64List self, SseSerializer serializer);
@protected
void sse_encode_list_prim_u_8_strict(
Uint8List self, SseSerializer serializer);
@protected
void sse_encode_record_list_list_prim_u_64_strict_list_list_prim_f_32_strict(
(List<Uint64List>, List<Float32List>) self, SseSerializer serializer);
@protected
void sse_encode_record_list_prim_u_64_strict_list_prim_f_32_strict(
(Uint64List, Float32List) self, SseSerializer serializer);
@protected
void sse_encode_record_usize_usize_usize_usize_usize_usize_usize(
(BigInt, BigInt, BigInt, BigInt, BigInt, BigInt, BigInt) self,
SseSerializer serializer);
@protected
void sse_encode_u_64(BigInt self, SseSerializer serializer);
@protected
void sse_encode_u_8(int self, SseSerializer serializer);
@protected
void sse_encode_unit(void self, SseSerializer serializer);
@protected
void sse_encode_usize(BigInt self, SseSerializer serializer);
@protected
void sse_encode_i_32(int self, SseSerializer serializer);
}
// Section: wire_class
class RustLibWire implements BaseWire {
factory RustLibWire.fromExternalLibrary(ExternalLibrary lib) =>
RustLibWire(lib.ffiDynamicLibrary);
/// Holds the symbol lookup function.
final ffi.Pointer<T> Function<T extends ffi.NativeType>(String symbolName)
_lookup;
/// The symbols are looked up in [dynamicLibrary].
RustLibWire(ffi.DynamicLibrary dynamicLibrary)
: _lookup = dynamicLibrary.lookup;
void
rust_arc_increment_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
ffi.Pointer<ffi.Void> ptr,
) {
return _rust_arc_increment_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
ptr,
);
}
late final _rust_arc_increment_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDBPtr =
_lookup<ffi.NativeFunction<ffi.Void Function(ffi.Pointer<ffi.Void>)>>(
'frbgen_photos_rust_arc_increment_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB');
late final _rust_arc_increment_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB =
_rust_arc_increment_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDBPtr
.asFunction<void Function(ffi.Pointer<ffi.Void>)>();
void
rust_arc_decrement_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
ffi.Pointer<ffi.Void> ptr,
) {
return _rust_arc_decrement_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB(
ptr,
);
}
late final _rust_arc_decrement_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDBPtr =
_lookup<ffi.NativeFunction<ffi.Void Function(ffi.Pointer<ffi.Void>)>>(
'frbgen_photos_rust_arc_decrement_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB');
late final _rust_arc_decrement_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDB =
_rust_arc_decrement_strong_count_RustOpaque_flutter_rust_bridgefor_generatedRustAutoOpaqueInnerVectorDBPtr
.asFunction<void Function(ffi.Pointer<ffi.Void>)>();
}

View File

@@ -18,8 +18,9 @@ class BottomOfTitleBarWidget extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Row(
mainAxisAlignment: showCloseButton ? MainAxisAlignment.spaceBetween :
MainAxisAlignment.start,
mainAxisAlignment: showCloseButton
? MainAxisAlignment.spaceBetween
: MainAxisAlignment.start,
children: [
Flexible(
child: Padding(

View File

@@ -100,7 +100,7 @@ class ExtentsPageView extends StatefulWidget {
int? itemCount,
this.dragStartBehavior = DragStartBehavior.start,
this.openDrawer,
}) : childrenDelegate = SliverChildBuilderDelegate(
}) : childrenDelegate = SliverChildBuilderDelegate(
itemBuilder,
childCount: itemCount,
addAutomaticKeepAlives: false,
@@ -198,7 +198,7 @@ class ExtentsPageView extends StatefulWidget {
required this.childrenDelegate,
this.dragStartBehavior = DragStartBehavior.start,
this.openDrawer,
}) : extents = 0;
}) : extents = 0;
/// The number of pages to build off screen.
///

View File

@@ -9,8 +9,7 @@ import "package:photos/utils/navigation_util.dart";
class HeaderErrorWidget extends StatelessWidget {
final Error? _error;
const HeaderErrorWidget({super.key, required Error? error})
: _error = error;
const HeaderErrorWidget({super.key, required Error? error}) : _error = error;
@override
Widget build(BuildContext context) {

View File

@@ -1,11 +1,20 @@
import "dart:async";
import "dart:math" show Random;
import "dart:typed_data" show Float32List;
import "package:flutter/foundation.dart" show kDebugMode;
import 'package:flutter/material.dart';
import "package:flutter_rust_bridge/flutter_rust_bridge.dart";
import "package:logging/logging.dart";
import "package:ml_linalg/linalg.dart";
import "package:path_provider/path_provider.dart";
import "package:photos/core/constants.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/db/ml/clip_vector_db.dart";
import "package:photos/db/ml/db.dart";
import "package:photos/events/people_changed_event.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/generated/protos/ente/common/vector.pb.dart";
import "package:photos/models/ml/face/person.dart";
import "package:photos/models/search/generic_search_result.dart";
import "package:photos/service_locator.dart";
@@ -15,6 +24,8 @@ import 'package:photos/services/machine_learning/ml_service.dart';
import "package:photos/services/machine_learning/semantic_search/semantic_search_service.dart";
import "package:photos/services/notification_service.dart";
import "package:photos/services/search_service.dart";
import "package:photos/src/rust/api/simple.dart";
import "package:photos/src/rust/api/usearch_api.dart";
import 'package:photos/theme/ente_theme.dart';
import 'package:photos/ui/components/captioned_text_widget.dart';
import 'package:photos/ui/components/expandable_menu_item_widget.dart';
@@ -184,6 +195,380 @@ class _MLDebugSectionWidgetState extends State<MLDebugSectionWidget> {
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Do some basic usearch",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
try {
// randomly generate some vectors and keys
final random = Random();
final tenEmbeddings = List.generate(
10,
(index) {
final randomList = List<double>.generate(
192,
(_) => random.nextDouble(), // Values between 0 and 1
);
final randomVector = Vector.fromList(randomList).normalize();
return Float32List.fromList(randomVector.toList());
},
);
final tenKeys = Uint64List.fromList(
List.generate(
tenEmbeddings.length,
(index) => BigInt.from(index + 1),
),
);
final embedDimensions = BigInt.from(tenEmbeddings.first.length);
final indexPath = (await getApplicationSupportDirectory()).path +
"/ml/test/vector_db_index.usearch";
final rustVectorDB = VectorDb(
filePath: indexPath,
dimensions: embedDimensions,
);
await rustVectorDB.resetIndex();
final stats = await rustVectorDB.getIndexStats();
logger.info("vector_db stats: $stats");
await rustVectorDB.bulkAddVectors(
keys: tenKeys,
vectors: tenEmbeddings,
);
final statsAgain = await rustVectorDB.getIndexStats();
logger.info("vector_db stats again: $statsAgain");
final size = statsAgain.$1;
final capacity = statsAgain.$2;
final dimensions = statsAgain.$3;
showShortToast(
context,
"Size: $size, Capacity: $capacity, Dimensions: $dimensions",
);
await rustVectorDB.deleteIndex();
} catch (e, s) {
logger.warning('Rust bridge failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Fill ClipVectorDB",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
try {
final allClip = await MLDataDB.instance.getAllClipVectors();
final allClipSmall = allClip.sublist(0, 15000);
showShortToast(context, "Got all embeddings");
logger.info("Got all embeddings");
final clipVectorDB = ClipVectorDB.instance;
await clipVectorDB.deleteAllEmbeddings();
logger.info("Clean vector DB");
final stats = await clipVectorDB.getIndexStats();
logger.info(stats.toString());
showShortToast(context, stats.toString());
final fileIDs = allClipSmall.map((e) => e.fileID).toList();
final embeddings = allClipSmall
.map((e) => Float32List.fromList(e.vector.toList()))
.toList();
showShortToast(context, "Reshaped embeddings data");
logger.info("Reshaped embeddings data");
final now = DateTime.now();
await clipVectorDB.bulkInsertEmbeddings(
fileIDs: fileIDs,
embeddings: embeddings,
);
final duration = DateTime.now().difference(now);
logger.info(
"ClipVectorDB bulk insert took ${duration.inMilliseconds} ms for ${fileIDs.length} embeddings",
);
final statsAfter = await clipVectorDB.getIndexStats();
logger.info(statsAfter.toString());
showShortToast(context, statsAfter.toString());
} catch (e, s) {
logger.warning('ClipVectorDB migration failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Migrate to ClipVectorDB",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
try {
await MLDataDB.instance.checkMigrateFillClipVectorDB();
showShortToast(context, "Migration done!");
} catch (e, s) {
logger.warning('ClipVectorDB migration failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Show ClipVectorDB stats",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
try {
final clipVectorDB = ClipVectorDB.instance;
final stats = await clipVectorDB.getIndexStats();
logger.info(stats.toString());
showShortToast(context, stats.toString());
} catch (e, s) {
logger.warning('ClipVectorDB stats failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Delete/Empty ClipVectorDB",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
try {
final clipVectorDB = ClipVectorDB.instance;
await clipVectorDB.deleteIndex();
} catch (e, s) {
logger.warning('ClipVectorDB cleanup failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Benchmark Vector DB Face",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
try {
final w = (kDebugMode ? EnteWatch('MLDebugSectionWidget') : null)
?..start();
final persons = await PersonService.instance.getPersons();
w?.log('get all persons for ${persons.length} persons');
String laurensID = '';
for (final person in persons) {
if (person.data.name.toLowerCase().contains('laurens')) {
laurensID = person.remoteID;
}
}
if (laurensID.isEmpty) {
throw Exception('Laurens not found');
}
final laurensFaceIDs =
await MLDataDB.instance.getFaceIDsForPerson(laurensID);
w?.log(
'getting all face ids for laurens (${laurensFaceIDs.length} faces)',
);
final laurensFaceIdToEmbeddingData = await MLDataDB.instance
.getFaceEmbeddingMapForFaces(laurensFaceIDs);
// Fill the vector DB with all embeddings
final laurensFaceIdToFloat32 = laurensFaceIdToEmbeddingData.map(
(key, value) => MapEntry(
key,
Float32List.fromList(EVector.fromBuffer(value).values),
),
);
final keys = Uint64List.fromList(
List.generate(
laurensFaceIdToFloat32.length,
(index) => BigInt.from(index + 1),
),
);
final vectorDB = VectorDb(
filePath: (await getApplicationSupportDirectory()).path +
"/ml/test/vector_db_face_index.usearch",
dimensions: BigInt.from(
laurensFaceIdToFloat32.values.first.length,
),
);
await vectorDB.resetIndex();
await vectorDB.bulkAddVectors(
keys: keys,
vectors: laurensFaceIdToFloat32.values.toList(),
);
// Benchmarking the vector DB
final queries = laurensFaceIdToFloat32.values.toList();
final count = BigInt.from(10);
w?.reset();
final (vectorKeys, distances) = await vectorDB.bulkSearchVectors(
queries: queries,
count: count,
);
w?.log(
'Done with ${queries.length * queries.length} (${queries.length} x ${queries.length}}) embeddings comparisons in vector DB',
);
logger.info(
'vector db results: ${vectorKeys.length} results, first: ${vectorKeys.first}, hundredth: ${vectorKeys[99]}',
);
// Benchmarking our own vector comparisons
final laurensFaceIdToEmbeddingVectors =
laurensFaceIdToEmbeddingData.map(
(key, value) => MapEntry(
key,
Vector.fromList(EVector.fromBuffer(value).values),
),
);
final faceVectors = laurensFaceIdToEmbeddingVectors.values;
w?.reset();
for (final faceVector in faceVectors) {
for (final otherFaceVector in faceVectors) {
final _ = 1 - faceVector.dot(otherFaceVector);
}
}
w?.log(
'Done with ${faceVectors.length * faceVectors.length} (${faceVectors.length} x ${faceVectors.length}}) embeddings comparisons in own method',
);
await vectorDB.deleteIndex();
} catch (e, s) {
logger.warning('vector DB search failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Benchmark Vector DB CLIP",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
try {
final w = (kDebugMode ? EnteWatch('MLDebugSectionWidget') : null)
?..start();
final clipEmbeddings = await mlDataDB.getAllClipVectors();
w?.log(
'getting all clip embeddings (${clipEmbeddings.length} embeddings)',
);
// Fill the vector DB with all embeddings
final clipFloat32 = clipEmbeddings
.map(
(value) => Float32List.fromList(value.vector.toList()),
)
.toList();
final keys = Uint64List.fromList(
List.generate(
clipFloat32.length,
(index) => BigInt.from(index + 1),
),
);
final vectorDB = VectorDb(
filePath: (await getApplicationSupportDirectory()).path +
"/ml/test/vector_db_clip_index.usearch",
dimensions: BigInt.from(
clipFloat32.first.length,
),
);
await vectorDB.resetIndex();
await vectorDB.bulkAddVectors(
keys: keys,
vectors: clipFloat32,
);
// Benchmarking the vector DB
final count = BigInt.from(10);
w?.reset();
final (vectorKeys, distances) = await vectorDB.bulkSearchVectors(
queries: clipFloat32,
count: count,
);
w?.log(
'Done with ${clipFloat32.length * clipFloat32.length} (${clipFloat32.length} x ${clipFloat32.length}}) embeddings comparisons in vector DB',
);
logger.info(
'vector db results: ${vectorKeys.length} results, first: ${vectorKeys.first} with distances ${distances.first}, hundredth: ${vectorKeys[99]} with distances ${distances[99]}',
);
// // Benchmarking our own vector comparisons
// final clipVectors = clipEmbeddings
// .map(
// (value) => value.vector,
// )
// .toList();
// w?.reset();
// int compared = 0;
// int ms = DateTime.now().millisecondsSinceEpoch;
// for (final faceVector in clipVectors) {
// for (final otherFaceVector in clipVectors) {
// final _ = 1 - faceVector.dot(otherFaceVector);
// }
// compared++;
// if (compared % 100 == 0) {
// final now = DateTime.now().millisecondsSinceEpoch;
// logger.info(
// 'Compared next 100 in ${now - ms} ms, progress: ($compared / ${clipVectors.length})',
// );
// ms = now;
// }
// }
// w?.log(
// 'Done with ${clipVectors.length * clipVectors.length} (${clipVectors.length} x ${clipVectors.length}}) embeddings comparisons in own method',
// );
await vectorDB.deleteIndex();
} catch (e, s) {
logger.warning('vector DB search failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Test rust bridge",
),
pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true,
onTap: () async {
try {
final String greetings = greet(name: "Tom");
const String expected = "Hello, Tom!";
assert(greetings == expected);
debugPrint("String from rust: $greetings");
showShortToast(context, greetings);
} catch (e, s) {
logger.warning('Rust bridge failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
},
),
sectionOptionSpacing,
MenuItemWidget(
captionedTextWidget: FutureBuilder<IndexStatus>(
future: getIndexStatus(),

View File

@@ -57,57 +57,69 @@ class _MLUserDeveloperOptionsState extends State<MLUserDeveloperOptions> {
),
),
const SizedBox(height: 48),
widget.mlIsEnabled ? ButtonWidget(
buttonType: ButtonType.neutral,
labelText: "Purge empty indices",
onTap: () async {
await deleteEmptyIndices(context);
},
) : const SizedBox(),
widget.mlIsEnabled ? const SizedBox(height: 24) : const SizedBox(),
widget.mlIsEnabled ? ButtonWidget(
buttonType: ButtonType.neutral,
labelText: "Reset all local ML",
onTap: () async {
await deleteAllLocalML(context);
},
) : const SizedBox(),
widget.mlIsEnabled ? const SizedBox(height: 24) : const SizedBox(),
widget.mlIsEnabled ? MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Remote fetch",
),
menuItemColor: colorScheme.fillFaint,
trailingWidget: ToggleSwitchWidget(
value: () => localSettings.remoteFetchEnabled,
onChanged: () async {
try {
await localSettings.toggleRemoteFetch();
_logger.info(
'Remote fetch is turned ${localSettings.remoteFetchEnabled ? 'on' : 'off'}',
);
if (mounted) {
setState(() {});
}
} catch (e, s) {
_logger.warning(
'Remote fetch toggle failed ',
e,
s,
);
await showGenericErrorDialog(
context: context,
error: e,
);
}
},
),
singleBorderRadius: 8,
alignCaptionedTextToLeft: true,
isBottomBorderRadiusRemoved: true,
isGestureDetectorDisabled: true,
) : const SizedBox(),
widget.mlIsEnabled ? const SizedBox(height: 24) : const SizedBox.shrink(),
widget.mlIsEnabled
? ButtonWidget(
buttonType: ButtonType.neutral,
labelText: "Purge empty indices",
onTap: () async {
await deleteEmptyIndices(context);
},
)
: const SizedBox(),
widget.mlIsEnabled
? const SizedBox(height: 24)
: const SizedBox(),
widget.mlIsEnabled
? ButtonWidget(
buttonType: ButtonType.neutral,
labelText: "Reset all local ML",
onTap: () async {
await deleteAllLocalML(context);
},
)
: const SizedBox(),
widget.mlIsEnabled
? const SizedBox(height: 24)
: const SizedBox(),
widget.mlIsEnabled
? MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget(
title: "Remote fetch",
),
menuItemColor: colorScheme.fillFaint,
trailingWidget: ToggleSwitchWidget(
value: () => localSettings.remoteFetchEnabled,
onChanged: () async {
try {
await localSettings.toggleRemoteFetch();
_logger.info(
'Remote fetch is turned ${localSettings.remoteFetchEnabled ? 'on' : 'off'}',
);
if (mounted) {
setState(() {});
}
} catch (e, s) {
_logger.warning(
'Remote fetch toggle failed ',
e,
s,
);
await showGenericErrorDialog(
context: context,
error: e,
);
}
},
),
singleBorderRadius: 8,
alignCaptionedTextToLeft: true,
isBottomBorderRadiusRemoved: true,
isGestureDetectorDisabled: true,
)
: const SizedBox(),
widget.mlIsEnabled
? const SizedBox(height: 24)
: const SizedBox.shrink(),
ButtonWidget(
buttonType: ButtonType.neutral,
labelText: "Load face detection model",

View File

@@ -1,4 +1,3 @@
import "package:flutter/material.dart";
import 'package:photos/models/file/file.dart';
import "package:photos/ui/viewer/file/zoomable_image.dart";

View File

@@ -1,7 +1,6 @@
import "dart:async";
import 'package:fast_base58/fast_base58.dart';
import "package:flutter/cupertino.dart";
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import "package:local_auth/local_auth.dart";

View File

@@ -3,7 +3,8 @@ import "package:flutter/material.dart";
import "package:photos/generated/l10n.dart";
import "package:photos/theme/ente_theme.dart";
Future<DateTime?> showDatePickerSheet(BuildContext context, {
Future<DateTime?> showDatePickerSheet(
BuildContext context, {
required DateTime initialDate,
DateTime? maxDate,
DateTime? minDate,

View File

@@ -1,4 +1,3 @@
import 'package:flutter/material.dart';
import "package:photos/generated/l10n.dart";
import 'package:photos/models/file/file.dart';

View File

@@ -5,6 +5,7 @@ import "package:photos/theme/ente_theme.dart";
import "package:photos/ui/components/info_item_widget.dart";
import "package:photos/ui/viewer/date/edit_date_sheet.dart";
import "package:photos/utils/standalone/date_time.dart";
class CreationTimeItem extends StatefulWidget {
final EnteFile file;
final int currentUserID;

View File

@@ -1,18 +1,16 @@
import "package:photos/core/configuration.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
Future<bool> isMeAssigned() async {
final personEntities = await PersonService.instance.getPersons();
final currentUserEmail = Configuration.instance.getEmail();
final personEntities = await PersonService.instance.getPersons();
final currentUserEmail = Configuration.instance.getEmail();
bool isAssigned = false;
for (final personEntity in personEntities) {
if (personEntity.data.email == currentUserEmail) {
isAssigned = true;
break;
}
bool isAssigned = false;
for (final personEntity in personEntities) {
if (personEntity.data.email == currentUserEmail) {
isAssigned = true;
break;
}
return isAssigned;
}
}
return isAssigned;
}

View File

@@ -23,7 +23,7 @@ import "package:photos/events/file_uploaded_event.dart";
import 'package:photos/events/files_updated_event.dart';
import 'package:photos/events/local_photos_updated_event.dart';
import 'package:photos/events/subscription_purchased_event.dart';
import 'package:photos/main.dart';
import "package:photos/main.dart";
import "package:photos/models/api/metadata.dart";
import "package:photos/models/backup/backup_item.dart";
import "package:photos/models/backup/backup_item_status.dart";

View File

@@ -58,6 +58,8 @@ class FlagService {
bool get enableMobMultiPart => flags.enableMobMultiPart || internalUser;
bool get enableVectorDb => flags.internalUser;
String get castUrl => flags.castUrl;
bool hasSyncedAccountFlags() {

View File

@@ -175,6 +175,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "2.4.1"
build_cli_annotations:
dependency: transitive
description:
name: build_cli_annotations
sha256: b59d2769769efd6c9ff6d4c4cede0be115a566afc591705c2040b707534b1172
url: "https://pub.dev"
source: hosted
version: "2.1.0"
build_config:
dependency: transitive
description:
@@ -1007,6 +1015,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "2.0.24"
flutter_rust_bridge:
dependency: "direct main"
description:
name: flutter_rust_bridge
sha256: "37ef40bc6f863652e865f0b2563ea07f0d3c58d8efad803cc01933a4b2ee067e"
url: "https://pub.dev"
source: hosted
version: "2.11.1"
flutter_secure_storage:
dependency: "direct main"
description:
@@ -2137,6 +2153,13 @@ packages:
url: "https://github.com/KasemJaffer/receive_sharing_intent.git"
source: git
version: "1.8.1"
rust_lib_photos:
dependency: "direct main"
description:
path: rust_builder
relative: true
source: path
version: "0.0.1"
rxdart:
dependency: transitive
description:

View File

@@ -12,11 +12,11 @@ description: ente photos application
# Read more about iOS versioning at
# https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
version: 1.1.54+1084
version: 1.1.63+1097
publish_to: none
environment:
sdk: ">=3.0.0 <4.0.0"
sdk: ">=3.3.0 <4.0.0"
dependencies:
adaptive_theme: ^3.1.0
@@ -95,6 +95,7 @@ dependencies:
flutter_map: ^6.2.0
flutter_map_marker_cluster: ^1.3.6
flutter_password_strength: ^0.1.6
flutter_rust_bridge: 2.11.1
# Do not upgrade this package unless this issue is resolved:
# https://github.com/juliansteenbakker/flutter_secure_storage/issues/870
# On v9.2.4, keys related to lockscreen persist even after reintsall. For context see:
@@ -177,6 +178,8 @@ dependencies:
git:
url: https://github.com/KasemJaffer/receive_sharing_intent.git
ref: 2cea396
rust_lib_photos:
path: rust_builder
screenshot: ^3.0.0
scrollable_positioned_list: ^0.3.5
sentry: ^8.14.1

1
mobile/apps/photos/rust/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
/target

901
mobile/apps/photos/rust/Cargo.lock generated Normal file
View File

@@ -0,0 +1,901 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "addr2line"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
dependencies = [
"gimli",
]
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aho-corasick"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
dependencies = [
"memchr",
]
[[package]]
name = "allo-isolate"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "449e356a4864c017286dbbec0e12767ea07efba29e3b7d984194c2a7ff3c4550"
dependencies = [
"anyhow",
"atomic",
"backtrace",
]
[[package]]
name = "android_log-sys"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84521a3cf562bc62942e294181d9eef17eb38ceb8c68677bc49f144e4c3d4f8d"
[[package]]
name = "android_logger"
version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbb4e440d04be07da1f1bf44fb4495ebd58669372fe0cffa6e48595ac5bd88a3"
dependencies = [
"android_log-sys",
"env_filter",
"log",
]
[[package]]
name = "anyhow"
version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
[[package]]
name = "atomic"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba"
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "backtrace"
version = "0.3.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
dependencies = [
"addr2line",
"cc",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "bitflags"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "build-target"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "832133bbabbbaa9fbdba793456a2827627a7d2b8fb96032fa1e7666d7895832b"
[[package]]
name = "bumpalo"
version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
[[package]]
name = "bytemuck"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "cc"
version = "1.0.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
dependencies = [
"libc",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "codespan-reporting"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
dependencies = [
"termcolor",
"unicode-width",
]
[[package]]
name = "console_error_panic_hook"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc"
dependencies = [
"cfg-if",
"wasm-bindgen",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "cxx"
version = "1.0.112"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58ab30434ea0ff6aa640a08dda5284026a366d47565496fd40b6cbfbdd7e31a2"
dependencies = [
"cc",
"cxxbridge-flags",
"cxxbridge-macro",
"link-cplusplus",
]
[[package]]
name = "cxx-build"
version = "1.0.112"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b649d7dfae8268450d53d109388b337b9352c7cba1fc10db4a1bc23c3dc189fb"
dependencies = [
"cc",
"codespan-reporting",
"once_cell",
"proc-macro2",
"quote",
"scratch",
"syn",
]
[[package]]
name = "cxxbridge-flags"
version = "1.0.112"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42281b20eba5218c539295c667c18e2f50211bb11902419194c6ed1ae808e547"
[[package]]
name = "cxxbridge-macro"
version = "1.0.112"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b45506e3c66512b0a65d291a6b452128b7b1dd9841e20d1e151addbd2c00ea50"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "dart-sys"
version = "4.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57967e4b200d767d091b961d6ab42cc7d0cc14fe9e052e75d0d3cf9eb732d895"
dependencies = [
"cc",
]
[[package]]
name = "dashmap"
version = "5.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
dependencies = [
"cfg-if",
"hashbrown",
"lock_api",
"once_cell",
"parking_lot_core",
]
[[package]]
name = "delegate-attr"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51aac4c99b2e6775164b412ea33ae8441b2fde2dbf05a20bc0052a63d08c475b"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]]
name = "env_filter"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
dependencies = [
"log",
"regex",
]
[[package]]
name = "flutter_rust_bridge"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dde126295b2acc5f0a712e265e91b6fdc0ed38767496483e592ae7134db83725"
dependencies = [
"allo-isolate",
"android_logger",
"anyhow",
"build-target",
"bytemuck",
"byteorder",
"console_error_panic_hook",
"dart-sys",
"delegate-attr",
"flutter_rust_bridge_macros",
"futures",
"js-sys",
"lazy_static",
"log",
"oslog",
"portable-atomic",
"threadpool",
"tokio",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "flutter_rust_bridge_macros"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f0420326b13675321b194928bb7830043b68cf8b810e1c651285c747abb080"
dependencies = [
"hex",
"md-5",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
name = "futures-core"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
[[package]]
name = "futures-executor"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"
[[package]]
name = "futures-macro"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures-sink"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817"
[[package]]
name = "futures-task"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"
[[package]]
name = "futures-util"
version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "gimli"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hermit-abi"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "js-sys"
version = "0.3.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
[[package]]
name = "link-cplusplus"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a6f6da007f968f9def0d65a05b187e2960183de70c160204ecfccf0ee330212"
dependencies = [
"cc",
]
[[package]]
name = "lock_api"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest",
]
[[package]]
name = "memchr"
version = "2.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
[[package]]
name = "miniz_oxide"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
dependencies = [
"adler",
]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "object"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "oslog"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80d2043d1f61d77cb2f4b1f7b7b2295f40507f5f8e9d1c8bf10a1ca5f97a3969"
dependencies = [
"cc",
"dashmap",
"log",
]
[[package]]
name = "parking_lot_core"
version = "0.9.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-targets",
]
[[package]]
name = "pin-project-lite"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
[[package]]
name = "pin-utils"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "portable-atomic"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
[[package]]
name = "proc-macro2"
version = "1.0.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
[[package]]
name = "rust_lib_photos"
version = "0.1.0"
dependencies = [
"flutter_rust_bridge",
"usearch",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "scratch"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f6280af86e5f559536da57a45ebc84948833b3bee313a7dd25232e09c878a52"
[[package]]
name = "slab"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
dependencies = [
"autocfg",
]
[[package]]
name = "smallvec"
version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
[[package]]
name = "syn"
version = "2.0.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "termcolor"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
dependencies = [
"winapi-util",
]
[[package]]
name = "threadpool"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
dependencies = [
"num_cpus",
]
[[package]]
name = "tokio"
version = "1.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"
dependencies = [
"backtrace",
"num_cpus",
"pin-project-lite",
]
[[package]]
name = "typenum"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "unicode-ident"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "unicode-width"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
[[package]]
name = "usearch"
version = "2.17.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "908331accde6ff6bfe83e1f2dfd4cc77343d107bfacecd2f19b7a14d87cdb2df"
dependencies = [
"cxx",
"cxx-build",
]
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasm-bindgen"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
dependencies = [
"cfg-if",
"js-sys",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
[[package]]
name = "web-sys"
version = "0.3.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "winapi-util"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

View File

@@ -0,0 +1,14 @@
[package]
name = "rust_lib_photos"
version = "0.1.0"
edition = "2021"
[lib]
crate-type = ["cdylib", "staticlib"]
[dependencies]
flutter_rust_bridge = "=2.11.1"
usearch = "2.17.11"
[lints.rust]
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(frb_expand)'] }

View File

@@ -0,0 +1,2 @@
pub mod simple;
pub mod usearch_api;

View File

@@ -0,0 +1,10 @@
#[flutter_rust_bridge::frb(sync)] // Synchronous mode for simplicity of the demo
pub fn greet(name: String) -> String {
format!("Hello, {name}!")
}
#[flutter_rust_bridge::frb(init)]
pub fn init_app() {
// Default utilities - feel free to customize
flutter_rust_bridge::setup_default_user_utils();
}

View File

@@ -0,0 +1,188 @@
use flutter_rust_bridge::frb;
use usearch::{Index, IndexOptions, MetricKind, ScalarKind};
use std::path::PathBuf;
#[frb(opaque)]
pub struct VectorDB {
index: Index,
path: PathBuf,
}
impl VectorDB {
#[frb(sync)]
pub fn new(file_path: &str, dimensions: usize) -> Self {
let path = PathBuf::from(file_path);
let file_exists = path.try_exists().unwrap_or(false);
let mut options = IndexOptions::default();
options.dimensions = dimensions;
options.metric = MetricKind::IP;
options.quantization = ScalarKind::F32;
options.connectivity = 0; // auto
options.expansion_add = 0; // auto
options.expansion_search = 0; // auto
let index = Index::new(&options).expect("Failed to create index");
index
.reserve(1000)
.expect("Failed to reserve space in index");
let db = Self { index, path };
if file_exists {
println!("Loading index from disk.");
db.index.load(file_path).expect("Failed to load index");
} else {
println!("Creating new index.");
db.save_index();
}
db
}
fn save_index(&self) {
// Ensure directory exists
if let Some(parent) = self.path.parent() {
std::fs::create_dir_all(parent).expect("Failed to create directory");
}
self.index
.save(self.path.to_str().expect("Invalid path"))
.expect("Failed to save index");
}
fn ensure_capacity(&self, margin: usize) {
let current_size = self.index.size();
let capacity = self.index.capacity();
if current_size + margin + 1000 >= capacity {
self.index
.reserve(current_size + margin)
.expect("Failed to reserve space in index");
}
}
pub fn add_vector(&self, key: u64, vector: &Vec<f32>) {
if self.contains_vector(key) {
self.remove_vector(key);
} else {
self.ensure_capacity(1);
}
self.index.add(key, vector).expect("Failed to add vector");
self.save_index();
}
pub fn bulk_add_vectors(&self, keys: Vec<u64>, vectors: &Vec<Vec<f32>>) {
self.ensure_capacity(keys.len());
for (key, vector) in keys.iter().zip(vectors.iter()) {
if self.contains_vector(*key) {
self.remove_vector(*key);
}
self.index
.add(*key, vector)
.expect("Failed to (bulk) add vector");
}
self.save_index();
}
pub fn search_vectors(&self, query: &Vec<f32>, count: usize) -> (Vec<u64>, Vec<f32>) {
let matches = self
.index
.search(query, count)
.expect("Failed to search vectors");
(matches.keys, matches.distances)
}
pub fn bulk_search_vectors(
&self,
queries: &Vec<Vec<f32>>,
count: usize,
) -> (Vec<Vec<u64>>, Vec<Vec<f32>>) {
let mut keys = Vec::new();
let mut distances = Vec::new();
for query in queries {
let (keys_result, distances_result) = self.search_vectors(query, count);
keys.push(keys_result);
distances.push(distances_result);
}
(keys, distances)
}
/// Check if a vector with the given key exists in the index.
/// `true` if the index contains the vector with the given key, `false` otherwise.
pub fn contains_vector(&self, key: u64) -> bool {
self.index.contains(key)
}
pub fn get_vector(&self, key: u64) -> Vec<f32> {
let mut vector: Vec<f32> = vec![0.0; self.index.dimensions()];
self.index
.get(key, &mut vector)
.expect("Failed to get vector");
vector
}
pub fn bulk_get_vectors(&self, keys: Vec<u64>) -> Vec<Vec<f32>> {
let mut vectors = Vec::new();
for key in keys {
let vector = self.get_vector(key);
vectors.push(vector);
}
vectors
}
pub fn remove_vector(&self, key: u64) -> usize {
let removed_count = self.index.remove(key).expect("Failed to remove vector");
self.save_index();
removed_count
}
pub fn bulk_remove_vectors(&self, keys: Vec<u64>) -> usize {
let mut removed_count = 0;
for key in keys {
removed_count += self
.index
.remove(key)
.expect("Failed to (bulk) remove vector");
}
self.save_index();
removed_count
}
pub fn reset_index(&self) {
self.index.reset().expect("Failed to reset index");
self.index
.reserve(1000)
.expect("Failed to reserve space in index");
self.save_index();
}
pub fn delete_index(self) {
if self.path.exists() {
std::fs::remove_file(&self.path).expect("Failed to delete index file");
} else {
println!("Index file does not exist.");
}
}
pub fn get_index_stats(&self) -> (usize, usize, usize, usize, usize, usize, usize) {
let size = self.index.size();
let capacity = self.index.capacity();
let dimensions = self.index.dimensions();
let file_size = self.index.serialized_length();
let memory_usage = self.index.memory_usage();
let expansion_add = self.index.expansion_add();
let expansion_search = self.index.expansion_search();
(
size,
capacity,
dimensions,
file_size,
memory_usage,
expansion_add,
expansion_search,
)
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,2 @@
pub mod api;
mod frb_generated;

View File

@@ -0,0 +1,29 @@
# Miscellaneous
*.class
*.log
*.pyc
*.swp
.DS_Store
.atom/
.buildlog/
.history
.svn/
migrate_working_dir/
# IntelliJ related
*.iml
*.ipr
*.iws
.idea/
# The .vscode folder contains launch configuration and tasks you configure in
# VS Code which you may wish to be included in version control, so this line
# is commented out by default.
#.vscode/
# Flutter/Dart/Pub related
# Libraries should not include pubspec.lock, per https://dart.dev/guides/libraries/private-files#pubspeclock.
/pubspec.lock
**/doc/api/
.dart_tool/
build/

View File

@@ -0,0 +1 @@
Please ignore this folder, which is just glue to build Rust with Flutter.

View File

@@ -0,0 +1,9 @@
*.iml
.gradle
/local.properties
/.idea/workspace.xml
/.idea/libraries
.DS_Store
/build
/captures
.cxx

View File

@@ -0,0 +1,56 @@
// The Android Gradle Plugin builds the native code with the Android NDK.
group 'com.flutter_rust_bridge.rust_lib_photos'
version '1.0'
buildscript {
repositories {
google()
mavenCentral()
}
dependencies {
// The Android Gradle Plugin knows how to build native code with the NDK.
classpath 'com.android.tools.build:gradle:7.3.0'
}
}
rootProject.allprojects {
repositories {
google()
mavenCentral()
}
}
apply plugin: 'com.android.library'
android {
if (project.android.hasProperty("namespace")) {
namespace 'com.flutter_rust_bridge.rust_lib_photos'
}
// Bumping the plugin compileSdkVersion requires all clients of this plugin
// to bump the version in their app.
compileSdkVersion 33
// Use the NDK version
// declared in /android/app/build.gradle file of the Flutter project.
// Replace it with a version number if this plugin requires a specfic NDK version.
// (e.g. ndkVersion "23.1.7779620")
ndkVersion android.ndkVersion
compileOptions {
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
defaultConfig {
minSdkVersion 19
}
}
apply from: "../cargokit/gradle/plugin.gradle"
cargokit {
manifestDir = "../../rust"
libname = "rust_lib_photos"
}

View File

@@ -0,0 +1 @@
rootProject.name = 'rust_lib_photos'

View File

@@ -0,0 +1,3 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.flutter_rust_bridge.rust_lib_photos">
</manifest>

View File

@@ -0,0 +1,4 @@
target
.dart_tool
*.iml
!pubspec.lock

View File

@@ -0,0 +1,42 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
Copyright 2022 Matej Knopp
================================================================================
MIT LICENSE
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
================================================================================
APACHE LICENSE, VERSION 2.0
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@@ -0,0 +1,11 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
Experimental repository to provide glue for seamlessly integrating cargo build
with flutter plugins and packages.
See https://matejknopp.com/post/flutter_plugin_in_rust_with_no_prebuilt_binaries/
for a tutorial on how to use Cargokit.
Example plugin available at https://github.com/irondash/hello_rust_ffi_plugin.

View File

@@ -0,0 +1,58 @@
#!/bin/sh
set -e
BASEDIR=$(dirname "$0")
# Workaround for https://github.com/dart-lang/pub/issues/4010
BASEDIR=$(cd "$BASEDIR" ; pwd -P)
# Remove XCode SDK from path. Otherwise this breaks tool compilation when building iOS project
NEW_PATH=`echo $PATH | tr ":" "\n" | grep -v "Contents/Developer/" | tr "\n" ":"`
export PATH=${NEW_PATH%?} # remove trailing :
env
# Platform name (macosx, iphoneos, iphonesimulator)
export CARGOKIT_DARWIN_PLATFORM_NAME=$PLATFORM_NAME
# Arctive architectures (arm64, armv7, x86_64), space separated.
export CARGOKIT_DARWIN_ARCHS=$ARCHS
# Current build configuration (Debug, Release)
export CARGOKIT_CONFIGURATION=$CONFIGURATION
# Path to directory containing Cargo.toml.
export CARGOKIT_MANIFEST_DIR=$PODS_TARGET_SRCROOT/$1
# Temporary directory for build artifacts.
export CARGOKIT_TARGET_TEMP_DIR=$TARGET_TEMP_DIR
# Output directory for final artifacts.
export CARGOKIT_OUTPUT_DIR=$PODS_CONFIGURATION_BUILD_DIR/$PRODUCT_NAME
# Directory to store built tool artifacts.
export CARGOKIT_TOOL_TEMP_DIR=$TARGET_TEMP_DIR/build_tool
# Directory inside root project. Not necessarily the top level directory of root project.
export CARGOKIT_ROOT_PROJECT_DIR=$SRCROOT
FLUTTER_EXPORT_BUILD_ENVIRONMENT=(
"$PODS_ROOT/../Flutter/ephemeral/flutter_export_environment.sh" # macOS
"$PODS_ROOT/../Flutter/flutter_export_environment.sh" # iOS
)
for path in "${FLUTTER_EXPORT_BUILD_ENVIRONMENT[@]}"
do
if [[ -f "$path" ]]; then
source "$path"
fi
done
sh "$BASEDIR/run_build_tool.sh" build-pod "$@"
# Make a symlink from built framework to phony file, which will be used as input to
# build script. This should force rebuild (podspec currently doesn't support alwaysOutOfDate
# attribute on custom build phase)
ln -fs "$OBJROOT/XCBuildData/build.db" "${BUILT_PRODUCTS_DIR}/cargokit_phony"
ln -fs "${BUILT_PRODUCTS_DIR}/${EXECUTABLE_PATH}" "${BUILT_PRODUCTS_DIR}/cargokit_phony_out"

View File

@@ -0,0 +1,5 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
A sample command-line application with an entrypoint in `bin/`, library code
in `lib/`, and example unit test in `test/`.

View File

@@ -0,0 +1,34 @@
# This is copied from Cargokit (which is the official way to use it currently)
# Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
# This file configures the static analysis results for your project (errors,
# warnings, and lints).
#
# This enables the 'recommended' set of lints from `package:lints`.
# This set helps identify many issues that may lead to problems when running
# or consuming Dart code, and enforces writing Dart using a single, idiomatic
# style and format.
#
# If you want a smaller set of lints you can change this to specify
# 'package:lints/core.yaml'. These are just the most critical lints
# (the recommended set includes the core lints).
# The core lints are also what is used by pub.dev for scoring packages.
include: package:lints/recommended.yaml
# Uncomment the following section to specify additional rules.
linter:
rules:
- prefer_relative_imports
- directives_ordering
# analyzer:
# exclude:
# - path/to/excluded/files/**
# For more information about the core and recommended set of lints, see
# https://dart.dev/go/core-lints
# For additional information about configuring this file, see
# https://dart.dev/guides/language/analysis-options

View File

@@ -0,0 +1,8 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'package:build_tool/build_tool.dart' as build_tool;
void main(List<String> arguments) {
build_tool.runMain(arguments);
}

View File

@@ -0,0 +1,8 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'src/build_tool.dart' as build_tool;
Future<void> runMain(List<String> args) async {
return build_tool.runMain(args);
}

View File

@@ -0,0 +1,195 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'dart:isolate';
import 'dart:math' as math;
import 'package:collection/collection.dart';
import 'package:path/path.dart' as path;
import 'package:version/version.dart';
import 'target.dart';
import 'util.dart';
class AndroidEnvironment {
AndroidEnvironment({
required this.sdkPath,
required this.ndkVersion,
required this.minSdkVersion,
required this.targetTempDir,
required this.target,
});
static void clangLinkerWrapper(List<String> args) {
final clang = Platform.environment['_CARGOKIT_NDK_LINK_CLANG'];
if (clang == null) {
throw Exception(
"cargo-ndk rustc linker: didn't find _CARGOKIT_NDK_LINK_CLANG env var");
}
final target = Platform.environment['_CARGOKIT_NDK_LINK_TARGET'];
if (target == null) {
throw Exception(
"cargo-ndk rustc linker: didn't find _CARGOKIT_NDK_LINK_TARGET env var");
}
runCommand(clang, [
target,
...args,
]);
}
/// Full path to Android SDK.
final String sdkPath;
/// Full version of Android NDK.
final String ndkVersion;
/// Minimum supported SDK version.
final int minSdkVersion;
/// Target directory for build artifacts.
final String targetTempDir;
/// Target being built.
final Target target;
bool ndkIsInstalled() {
final ndkPath = path.join(sdkPath, 'ndk', ndkVersion);
final ndkPackageXml = File(path.join(ndkPath, 'package.xml'));
return ndkPackageXml.existsSync();
}
void installNdk({
required String javaHome,
}) {
final sdkManagerExtension = Platform.isWindows ? '.bat' : '';
final sdkManager = path.join(
sdkPath,
'cmdline-tools',
'latest',
'bin',
'sdkmanager$sdkManagerExtension',
);
log.info('Installing NDK $ndkVersion');
runCommand(sdkManager, [
'--install',
'ndk;$ndkVersion',
], environment: {
'JAVA_HOME': javaHome,
});
}
Future<Map<String, String>> buildEnvironment() async {
final hostArch = Platform.isMacOS
? "darwin-x86_64"
: (Platform.isLinux ? "linux-x86_64" : "windows-x86_64");
final ndkPath = path.join(sdkPath, 'ndk', ndkVersion);
final toolchainPath = path.join(
ndkPath,
'toolchains',
'llvm',
'prebuilt',
hostArch,
'bin',
);
final minSdkVersion =
math.max(target.androidMinSdkVersion!, this.minSdkVersion);
final exe = Platform.isWindows ? '.exe' : '';
final arKey = 'AR_${target.rust}';
final arValue = ['${target.rust}-ar', 'llvm-ar', 'llvm-ar.exe']
.map((e) => path.join(toolchainPath, e))
.firstWhereOrNull((element) => File(element).existsSync());
if (arValue == null) {
throw Exception('Failed to find ar for $target in $toolchainPath');
}
final targetArg = '--target=${target.rust}$minSdkVersion';
final ccKey = 'CC_${target.rust}';
final ccValue = path.join(toolchainPath, 'clang$exe');
final cfFlagsKey = 'CFLAGS_${target.rust}';
final cFlagsValue = targetArg;
final cxxKey = 'CXX_${target.rust}';
final cxxValue = path.join(toolchainPath, 'clang++$exe');
final cxxFlagsKey = 'CXXFLAGS_${target.rust}';
final cxxFlagsValue = targetArg;
final linkerKey =
'cargo_target_${target.rust.replaceAll('-', '_')}_linker'.toUpperCase();
final ranlibKey = 'RANLIB_${target.rust}';
final ranlibValue = path.join(toolchainPath, 'llvm-ranlib$exe');
final ndkVersionParsed = Version.parse(ndkVersion);
final rustFlagsKey = 'CARGO_ENCODED_RUSTFLAGS';
final rustFlagsValue = _libGccWorkaround(targetTempDir, ndkVersionParsed);
final runRustTool =
Platform.isWindows ? 'run_build_tool.cmd' : 'run_build_tool.sh';
final packagePath = (await Isolate.resolvePackageUri(
Uri.parse('package:build_tool/buildtool.dart')))!
.toFilePath();
final selfPath = path.canonicalize(path.join(
packagePath,
'..',
'..',
'..',
runRustTool,
));
// Make sure that run_build_tool is working properly even initially launched directly
// through dart run.
final toolTempDir =
Platform.environment['CARGOKIT_TOOL_TEMP_DIR'] ?? targetTempDir;
return {
arKey: arValue,
ccKey: ccValue,
cfFlagsKey: cFlagsValue,
cxxKey: cxxValue,
cxxFlagsKey: cxxFlagsValue,
ranlibKey: ranlibValue,
rustFlagsKey: rustFlagsValue,
linkerKey: selfPath,
// Recognized by main() so we know when we're acting as a wrapper
'_CARGOKIT_NDK_LINK_TARGET': targetArg,
'_CARGOKIT_NDK_LINK_CLANG': ccValue,
'CARGOKIT_TOOL_TEMP_DIR': toolTempDir,
};
}
// Workaround for libgcc missing in NDK23, inspired by cargo-ndk
String _libGccWorkaround(String buildDir, Version ndkVersion) {
final workaroundDir = path.join(
buildDir,
'cargokit',
'libgcc_workaround',
'${ndkVersion.major}',
);
Directory(workaroundDir).createSync(recursive: true);
if (ndkVersion.major >= 23) {
File(path.join(workaroundDir, 'libgcc.a'))
.writeAsStringSync('INPUT(-lunwind)');
} else {
// Other way around, untested, forward libgcc.a from libunwind once Rust
// gets updated for NDK23+.
File(path.join(workaroundDir, 'libunwind.a'))
.writeAsStringSync('INPUT(-lgcc)');
}
var rustFlags = Platform.environment['CARGO_ENCODED_RUSTFLAGS'] ?? '';
if (rustFlags.isNotEmpty) {
rustFlags = '$rustFlags\x1f';
}
rustFlags = '$rustFlags-L\x1f$workaroundDir';
return rustFlags;
}
}

View File

@@ -0,0 +1,266 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:ed25519_edwards/ed25519_edwards.dart';
import 'package:http/http.dart';
import 'package:logging/logging.dart';
import 'package:path/path.dart' as path;
import 'builder.dart';
import 'crate_hash.dart';
import 'options.dart';
import 'precompile_binaries.dart';
import 'rustup.dart';
import 'target.dart';
class Artifact {
/// File system location of the artifact.
final String path;
/// Actual file name that the artifact should have in destination folder.
final String finalFileName;
AritifactType get type {
if (finalFileName.endsWith('.dll') ||
finalFileName.endsWith('.dll.lib') ||
finalFileName.endsWith('.pdb') ||
finalFileName.endsWith('.so') ||
finalFileName.endsWith('.dylib')) {
return AritifactType.dylib;
} else if (finalFileName.endsWith('.lib') || finalFileName.endsWith('.a')) {
return AritifactType.staticlib;
} else {
throw Exception('Unknown artifact type for $finalFileName');
}
}
Artifact({
required this.path,
required this.finalFileName,
});
}
final _log = Logger('artifacts_provider');
class ArtifactProvider {
ArtifactProvider({
required this.environment,
required this.userOptions,
});
final BuildEnvironment environment;
final CargokitUserOptions userOptions;
Future<Map<Target, List<Artifact>>> getArtifacts(List<Target> targets) async {
final result = await _getPrecompiledArtifacts(targets);
final pendingTargets = List.of(targets);
pendingTargets.removeWhere((element) => result.containsKey(element));
if (pendingTargets.isEmpty) {
return result;
}
final rustup = Rustup();
for (final target in targets) {
final builder = RustBuilder(target: target, environment: environment);
builder.prepare(rustup);
_log.info('Building ${environment.crateInfo.packageName} for $target');
final targetDir = await builder.build();
// For local build accept both static and dynamic libraries.
final artifactNames = <String>{
...getArtifactNames(
target: target,
libraryName: environment.crateInfo.packageName,
aritifactType: AritifactType.dylib,
remote: false,
),
...getArtifactNames(
target: target,
libraryName: environment.crateInfo.packageName,
aritifactType: AritifactType.staticlib,
remote: false,
)
};
final artifacts = artifactNames
.map((artifactName) => Artifact(
path: path.join(targetDir, artifactName),
finalFileName: artifactName,
))
.where((element) => File(element.path).existsSync())
.toList();
result[target] = artifacts;
}
return result;
}
Future<Map<Target, List<Artifact>>> _getPrecompiledArtifacts(
List<Target> targets) async {
if (userOptions.usePrecompiledBinaries == false) {
_log.info('Precompiled binaries are disabled');
return {};
}
if (environment.crateOptions.precompiledBinaries == null) {
_log.fine('Precompiled binaries not enabled for this crate');
return {};
}
final start = Stopwatch()..start();
final crateHash = CrateHash.compute(environment.manifestDir,
tempStorage: environment.targetTempDir);
_log.fine(
'Computed crate hash $crateHash in ${start.elapsedMilliseconds}ms');
final downloadedArtifactsDir =
path.join(environment.targetTempDir, 'precompiled', crateHash);
Directory(downloadedArtifactsDir).createSync(recursive: true);
final res = <Target, List<Artifact>>{};
for (final target in targets) {
final requiredArtifacts = getArtifactNames(
target: target,
libraryName: environment.crateInfo.packageName,
remote: true,
);
final artifactsForTarget = <Artifact>[];
for (final artifact in requiredArtifacts) {
final fileName = PrecompileBinaries.fileName(target, artifact);
final downloadedPath = path.join(downloadedArtifactsDir, fileName);
if (!File(downloadedPath).existsSync()) {
final signatureFileName =
PrecompileBinaries.signatureFileName(target, artifact);
await _tryDownloadArtifacts(
crateHash: crateHash,
fileName: fileName,
signatureFileName: signatureFileName,
finalPath: downloadedPath,
);
}
if (File(downloadedPath).existsSync()) {
artifactsForTarget.add(Artifact(
path: downloadedPath,
finalFileName: artifact,
));
} else {
break;
}
}
// Only provide complete set of artifacts.
if (artifactsForTarget.length == requiredArtifacts.length) {
_log.fine('Found precompiled artifacts for $target');
res[target] = artifactsForTarget;
}
}
return res;
}
static Future<Response> _get(Uri url, {Map<String, String>? headers}) async {
int attempt = 0;
const maxAttempts = 10;
while (true) {
try {
return await get(url, headers: headers);
} on SocketException catch (e) {
// Try to detect reset by peer error and retry.
if (attempt++ < maxAttempts &&
(e.osError?.errorCode == 54 || e.osError?.errorCode == 10054)) {
_log.severe(
'Failed to download $url: $e, attempt $attempt of $maxAttempts, will retry...');
await Future.delayed(Duration(seconds: 1));
continue;
} else {
rethrow;
}
}
}
}
Future<void> _tryDownloadArtifacts({
required String crateHash,
required String fileName,
required String signatureFileName,
required String finalPath,
}) async {
final precompiledBinaries = environment.crateOptions.precompiledBinaries!;
final prefix = precompiledBinaries.uriPrefix;
final url = Uri.parse('$prefix$crateHash/$fileName');
final signatureUrl = Uri.parse('$prefix$crateHash/$signatureFileName');
_log.fine('Downloading signature from $signatureUrl');
final signature = await _get(signatureUrl);
if (signature.statusCode == 404) {
_log.warning(
'Precompiled binaries not available for crate hash $crateHash ($fileName)');
return;
}
if (signature.statusCode != 200) {
_log.severe(
'Failed to download signature $signatureUrl: status ${signature.statusCode}');
return;
}
_log.fine('Downloading binary from $url');
final res = await _get(url);
if (res.statusCode != 200) {
_log.severe('Failed to download binary $url: status ${res.statusCode}');
return;
}
if (verify(
precompiledBinaries.publicKey, res.bodyBytes, signature.bodyBytes)) {
File(finalPath).writeAsBytesSync(res.bodyBytes);
} else {
_log.shout('Signature verification failed! Ignoring binary.');
}
}
}
enum AritifactType {
staticlib,
dylib,
}
AritifactType artifactTypeForTarget(Target target) {
if (target.darwinPlatform != null) {
return AritifactType.staticlib;
} else {
return AritifactType.dylib;
}
}
List<String> getArtifactNames({
required Target target,
required String libraryName,
required bool remote,
AritifactType? aritifactType,
}) {
aritifactType ??= artifactTypeForTarget(target);
if (target.darwinArch != null) {
if (aritifactType == AritifactType.staticlib) {
return ['lib$libraryName.a'];
} else {
return ['lib$libraryName.dylib'];
}
} else if (target.rust.contains('-windows-')) {
if (aritifactType == AritifactType.staticlib) {
return ['$libraryName.lib'];
} else {
return [
'$libraryName.dll',
'$libraryName.dll.lib',
if (!remote) '$libraryName.pdb'
];
}
} else if (target.rust.contains('-linux-')) {
if (aritifactType == AritifactType.staticlib) {
return ['lib$libraryName.a'];
} else {
return ['lib$libraryName.so'];
}
} else {
throw Exception("Unsupported target: ${target.rust}");
}
}

View File

@@ -0,0 +1,40 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:path/path.dart' as path;
import 'artifacts_provider.dart';
import 'builder.dart';
import 'environment.dart';
import 'options.dart';
import 'target.dart';
class BuildCMake {
final CargokitUserOptions userOptions;
BuildCMake({required this.userOptions});
Future<void> build() async {
final targetPlatform = Environment.targetPlatform;
final target = Target.forFlutterName(Environment.targetPlatform);
if (target == null) {
throw Exception("Unknown target platform: $targetPlatform");
}
final environment = BuildEnvironment.fromEnvironment(isAndroid: false);
final provider =
ArtifactProvider(environment: environment, userOptions: userOptions);
final artifacts = await provider.getArtifacts([target]);
final libs = artifacts[target]!;
for (final lib in libs) {
if (lib.type == AritifactType.dylib) {
File(lib.path)
.copySync(path.join(Environment.outputDir, lib.finalFileName));
}
}
}
}

View File

@@ -0,0 +1,49 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:logging/logging.dart';
import 'package:path/path.dart' as path;
import 'artifacts_provider.dart';
import 'builder.dart';
import 'environment.dart';
import 'options.dart';
import 'target.dart';
final log = Logger('build_gradle');
class BuildGradle {
BuildGradle({required this.userOptions});
final CargokitUserOptions userOptions;
Future<void> build() async {
final targets = Environment.targetPlatforms.map((arch) {
final target = Target.forFlutterName(arch);
if (target == null) {
throw Exception(
"Unknown darwin target or platform: $arch, ${Environment.darwinPlatformName}");
}
return target;
}).toList();
final environment = BuildEnvironment.fromEnvironment(isAndroid: true);
final provider =
ArtifactProvider(environment: environment, userOptions: userOptions);
final artifacts = await provider.getArtifacts(targets);
for (final target in targets) {
final libs = artifacts[target]!;
final outputDir = path.join(Environment.outputDir, target.android!);
Directory(outputDir).createSync(recursive: true);
for (final lib in libs) {
if (lib.type == AritifactType.dylib) {
File(lib.path).copySync(path.join(outputDir, lib.finalFileName));
}
}
}
}
}

View File

@@ -0,0 +1,89 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:path/path.dart' as path;
import 'artifacts_provider.dart';
import 'builder.dart';
import 'environment.dart';
import 'options.dart';
import 'target.dart';
import 'util.dart';
class BuildPod {
BuildPod({required this.userOptions});
final CargokitUserOptions userOptions;
Future<void> build() async {
final targets = Environment.darwinArchs.map((arch) {
final target = Target.forDarwin(
platformName: Environment.darwinPlatformName, darwinAarch: arch);
if (target == null) {
throw Exception(
"Unknown darwin target or platform: $arch, ${Environment.darwinPlatformName}");
}
return target;
}).toList();
final environment = BuildEnvironment.fromEnvironment(isAndroid: false);
final provider =
ArtifactProvider(environment: environment, userOptions: userOptions);
final artifacts = await provider.getArtifacts(targets);
void performLipo(String targetFile, Iterable<String> sourceFiles) {
runCommand("lipo", [
'-create',
...sourceFiles,
'-output',
targetFile,
]);
}
final outputDir = Environment.outputDir;
Directory(outputDir).createSync(recursive: true);
final staticLibs = artifacts.values
.expand((element) => element)
.where((element) => element.type == AritifactType.staticlib)
.toList();
final dynamicLibs = artifacts.values
.expand((element) => element)
.where((element) => element.type == AritifactType.dylib)
.toList();
final libName = environment.crateInfo.packageName;
// If there is static lib, use it and link it with pod
if (staticLibs.isNotEmpty) {
final finalTargetFile = path.join(outputDir, "lib$libName.a");
performLipo(finalTargetFile, staticLibs.map((e) => e.path));
} else {
// Otherwise try to replace bundle dylib with our dylib
final bundlePaths = [
'$libName.framework/Versions/A/$libName',
'$libName.framework/$libName',
];
for (final bundlePath in bundlePaths) {
final targetFile = path.join(outputDir, bundlePath);
if (File(targetFile).existsSync()) {
performLipo(targetFile, dynamicLibs.map((e) => e.path));
// Replace absolute id with @rpath one so that it works properly
// when moved to Frameworks.
runCommand("install_name_tool", [
'-id',
'@rpath/$bundlePath',
targetFile,
]);
return;
}
}
throw Exception('Unable to find bundle for dynamic library');
}
}
}

View File

@@ -0,0 +1,271 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:ed25519_edwards/ed25519_edwards.dart';
import 'package:github/github.dart';
import 'package:hex/hex.dart';
import 'package:logging/logging.dart';
import 'android_environment.dart';
import 'build_cmake.dart';
import 'build_gradle.dart';
import 'build_pod.dart';
import 'logging.dart';
import 'options.dart';
import 'precompile_binaries.dart';
import 'target.dart';
import 'util.dart';
import 'verify_binaries.dart';
final log = Logger('build_tool');
abstract class BuildCommand extends Command {
Future<void> runBuildCommand(CargokitUserOptions options);
@override
Future<void> run() async {
final options = CargokitUserOptions.load();
if (options.verboseLogging ||
Platform.environment['CARGOKIT_VERBOSE'] == '1') {
enableVerboseLogging();
}
await runBuildCommand(options);
}
}
class BuildPodCommand extends BuildCommand {
@override
final name = 'build-pod';
@override
final description = 'Build cocoa pod library';
@override
Future<void> runBuildCommand(CargokitUserOptions options) async {
final build = BuildPod(userOptions: options);
await build.build();
}
}
class BuildGradleCommand extends BuildCommand {
@override
final name = 'build-gradle';
@override
final description = 'Build android library';
@override
Future<void> runBuildCommand(CargokitUserOptions options) async {
final build = BuildGradle(userOptions: options);
await build.build();
}
}
class BuildCMakeCommand extends BuildCommand {
@override
final name = 'build-cmake';
@override
final description = 'Build CMake library';
@override
Future<void> runBuildCommand(CargokitUserOptions options) async {
final build = BuildCMake(userOptions: options);
await build.build();
}
}
class GenKeyCommand extends Command {
@override
final name = 'gen-key';
@override
final description = 'Generate key pair for signing precompiled binaries';
@override
void run() {
final kp = generateKey();
final private = HEX.encode(kp.privateKey.bytes);
final public = HEX.encode(kp.publicKey.bytes);
print("Private Key: $private");
print("Public Key: $public");
}
}
class PrecompileBinariesCommand extends Command {
PrecompileBinariesCommand() {
argParser
..addOption(
'repository',
mandatory: true,
help: 'Github repository slug in format owner/name',
)
..addOption(
'manifest-dir',
mandatory: true,
help: 'Directory containing Cargo.toml',
)
..addMultiOption('target',
help: 'Rust target triple of artifact to build.\n'
'Can be specified multiple times or omitted in which case\n'
'all targets for current platform will be built.')
..addOption(
'android-sdk-location',
help: 'Location of Android SDK (if available)',
)
..addOption(
'android-ndk-version',
help: 'Android NDK version (if available)',
)
..addOption(
'android-min-sdk-version',
help: 'Android minimum rquired version (if available)',
)
..addOption(
'temp-dir',
help: 'Directory to store temporary build artifacts',
)
..addFlag(
"verbose",
abbr: "v",
defaultsTo: false,
help: "Enable verbose logging",
);
}
@override
final name = 'precompile-binaries';
@override
final description = 'Prebuild and upload binaries\n'
'Private key must be passed through PRIVATE_KEY environment variable. '
'Use gen_key through generate priave key.\n'
'Github token must be passed as GITHUB_TOKEN environment variable.\n';
@override
Future<void> run() async {
final verbose = argResults!['verbose'] as bool;
if (verbose) {
enableVerboseLogging();
}
final privateKeyString = Platform.environment['PRIVATE_KEY'];
if (privateKeyString == null) {
throw ArgumentError('Missing PRIVATE_KEY environment variable');
}
final githubToken = Platform.environment['GITHUB_TOKEN'];
if (githubToken == null) {
throw ArgumentError('Missing GITHUB_TOKEN environment variable');
}
final privateKey = HEX.decode(privateKeyString);
if (privateKey.length != 64) {
throw ArgumentError('Private key must be 64 bytes long');
}
final manifestDir = argResults!['manifest-dir'] as String;
if (!Directory(manifestDir).existsSync()) {
throw ArgumentError('Manifest directory does not exist: $manifestDir');
}
String? androidMinSdkVersionString =
argResults!['android-min-sdk-version'] as String?;
int? androidMinSdkVersion;
if (androidMinSdkVersionString != null) {
androidMinSdkVersion = int.tryParse(androidMinSdkVersionString);
if (androidMinSdkVersion == null) {
throw ArgumentError(
'Invalid android-min-sdk-version: $androidMinSdkVersionString');
}
}
final targetStrigns = argResults!['target'] as List<String>;
final targets = targetStrigns.map((target) {
final res = Target.forRustTriple(target);
if (res == null) {
throw ArgumentError('Invalid target: $target');
}
return res;
}).toList(growable: false);
final precompileBinaries = PrecompileBinaries(
privateKey: PrivateKey(privateKey),
githubToken: githubToken,
manifestDir: manifestDir,
repositorySlug: RepositorySlug.full(argResults!['repository'] as String),
targets: targets,
androidSdkLocation: argResults!['android-sdk-location'] as String?,
androidNdkVersion: argResults!['android-ndk-version'] as String?,
androidMinSdkVersion: androidMinSdkVersion,
tempDir: argResults!['temp-dir'] as String?,
);
await precompileBinaries.run();
}
}
class VerifyBinariesCommand extends Command {
VerifyBinariesCommand() {
argParser.addOption(
'manifest-dir',
mandatory: true,
help: 'Directory containing Cargo.toml',
);
}
@override
final name = "verify-binaries";
@override
final description = 'Verifies published binaries\n'
'Checks whether there is a binary published for each targets\n'
'and checks the signature.';
@override
Future<void> run() async {
final manifestDir = argResults!['manifest-dir'] as String;
final verifyBinaries = VerifyBinaries(
manifestDir: manifestDir,
);
await verifyBinaries.run();
}
}
Future<void> runMain(List<String> args) async {
try {
// Init logging before options are loaded
initLogging();
if (Platform.environment['_CARGOKIT_NDK_LINK_TARGET'] != null) {
return AndroidEnvironment.clangLinkerWrapper(args);
}
final runner = CommandRunner('build_tool', 'Cargokit built_tool')
..addCommand(BuildPodCommand())
..addCommand(BuildGradleCommand())
..addCommand(BuildCMakeCommand())
..addCommand(GenKeyCommand())
..addCommand(PrecompileBinariesCommand())
..addCommand(VerifyBinariesCommand());
await runner.run(args);
} on ArgumentError catch (e) {
stderr.writeln(e.toString());
exit(1);
} catch (e, s) {
log.severe(kDoubleSeparator);
log.severe('Cargokit BuildTool failed with error:');
log.severe(kSeparator);
log.severe(e);
// This tells user to install Rust, there's no need to pollute the log with
// stack trace.
if (e is! RustupNotFoundException) {
log.severe(kSeparator);
log.severe(s);
log.severe(kSeparator);
log.severe('BuildTool arguments: $args');
}
log.severe(kDoubleSeparator);
exit(1);
}
}

View File

@@ -0,0 +1,198 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'package:collection/collection.dart';
import 'package:logging/logging.dart';
import 'package:path/path.dart' as path;
import 'android_environment.dart';
import 'cargo.dart';
import 'environment.dart';
import 'options.dart';
import 'rustup.dart';
import 'target.dart';
import 'util.dart';
final _log = Logger('builder');
enum BuildConfiguration {
debug,
release,
profile,
}
extension on BuildConfiguration {
bool get isDebug => this == BuildConfiguration.debug;
String get rustName => switch (this) {
BuildConfiguration.debug => 'debug',
BuildConfiguration.release => 'release',
BuildConfiguration.profile => 'release',
};
}
class BuildException implements Exception {
final String message;
BuildException(this.message);
@override
String toString() {
return 'BuildException: $message';
}
}
class BuildEnvironment {
final BuildConfiguration configuration;
final CargokitCrateOptions crateOptions;
final String targetTempDir;
final String manifestDir;
final CrateInfo crateInfo;
final bool isAndroid;
final String? androidSdkPath;
final String? androidNdkVersion;
final int? androidMinSdkVersion;
final String? javaHome;
BuildEnvironment({
required this.configuration,
required this.crateOptions,
required this.targetTempDir,
required this.manifestDir,
required this.crateInfo,
required this.isAndroid,
this.androidSdkPath,
this.androidNdkVersion,
this.androidMinSdkVersion,
this.javaHome,
});
static BuildConfiguration parseBuildConfiguration(String value) {
// XCode configuration adds the flavor to configuration name.
final firstSegment = value.split('-').first;
final buildConfiguration = BuildConfiguration.values.firstWhereOrNull(
(e) => e.name == firstSegment,
);
if (buildConfiguration == null) {
_log.warning('Unknown build configuraiton $value, will assume release');
return BuildConfiguration.release;
}
return buildConfiguration;
}
static BuildEnvironment fromEnvironment({
required bool isAndroid,
}) {
final buildConfiguration =
parseBuildConfiguration(Environment.configuration);
final manifestDir = Environment.manifestDir;
final crateOptions = CargokitCrateOptions.load(
manifestDir: manifestDir,
);
final crateInfo = CrateInfo.load(manifestDir);
return BuildEnvironment(
configuration: buildConfiguration,
crateOptions: crateOptions,
targetTempDir: Environment.targetTempDir,
manifestDir: manifestDir,
crateInfo: crateInfo,
isAndroid: isAndroid,
androidSdkPath: isAndroid ? Environment.sdkPath : null,
androidNdkVersion: isAndroid ? Environment.ndkVersion : null,
androidMinSdkVersion:
isAndroid ? int.parse(Environment.minSdkVersion) : null,
javaHome: isAndroid ? Environment.javaHome : null,
);
}
}
class RustBuilder {
final Target target;
final BuildEnvironment environment;
RustBuilder({
required this.target,
required this.environment,
});
void prepare(
Rustup rustup,
) {
final toolchain = _toolchain;
if (rustup.installedTargets(toolchain) == null) {
rustup.installToolchain(toolchain);
}
if (toolchain == 'nightly') {
rustup.installRustSrcForNightly();
}
if (!rustup.installedTargets(toolchain)!.contains(target.rust)) {
rustup.installTarget(target.rust, toolchain: toolchain);
}
}
CargoBuildOptions? get _buildOptions =>
environment.crateOptions.cargo[environment.configuration];
String get _toolchain => _buildOptions?.toolchain.name ?? 'stable';
/// Returns the path of directory containing build artifacts.
Future<String> build() async {
final extraArgs = _buildOptions?.flags ?? [];
final manifestPath = path.join(environment.manifestDir, 'Cargo.toml');
runCommand(
'rustup',
[
'run',
_toolchain,
'cargo',
'build',
...extraArgs,
'--manifest-path',
manifestPath,
'-p',
environment.crateInfo.packageName,
if (!environment.configuration.isDebug) '--release',
'--target',
target.rust,
'--target-dir',
environment.targetTempDir,
],
environment: await _buildEnvironment(),
);
return path.join(
environment.targetTempDir,
target.rust,
environment.configuration.rustName,
);
}
Future<Map<String, String>> _buildEnvironment() async {
if (target.android == null) {
return {};
} else {
final sdkPath = environment.androidSdkPath;
final ndkVersion = environment.androidNdkVersion;
final minSdkVersion = environment.androidMinSdkVersion;
if (sdkPath == null) {
throw BuildException('androidSdkPath is not set');
}
if (ndkVersion == null) {
throw BuildException('androidNdkVersion is not set');
}
if (minSdkVersion == null) {
throw BuildException('androidMinSdkVersion is not set');
}
final env = AndroidEnvironment(
sdkPath: sdkPath,
ndkVersion: ndkVersion,
minSdkVersion: minSdkVersion,
targetTempDir: environment.targetTempDir,
target: target,
);
if (!env.ndkIsInstalled() && environment.javaHome != null) {
env.installNdk(javaHome: environment.javaHome!);
}
return env.buildEnvironment();
}
}
}

View File

@@ -0,0 +1,48 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:path/path.dart' as path;
import 'package:toml/toml.dart';
class ManifestException {
ManifestException(this.message, {required this.fileName});
final String? fileName;
final String message;
@override
String toString() {
if (fileName != null) {
return 'Failed to parse package manifest at $fileName: $message';
} else {
return 'Failed to parse package manifest: $message';
}
}
}
class CrateInfo {
CrateInfo({required this.packageName});
final String packageName;
static CrateInfo parseManifest(String manifest, {final String? fileName}) {
final toml = TomlDocument.parse(manifest);
final package = toml.toMap()['package'];
if (package == null) {
throw ManifestException('Missing package section', fileName: fileName);
}
final name = package['name'];
if (name == null) {
throw ManifestException('Missing package name', fileName: fileName);
}
return CrateInfo(packageName: name);
}
static CrateInfo load(String manifestDir) {
final manifestFile = File(path.join(manifestDir, 'Cargo.toml'));
final manifest = manifestFile.readAsStringSync();
return parseManifest(manifest, fileName: manifestFile.path);
}
}

View File

@@ -0,0 +1,124 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:collection/collection.dart';
import 'package:convert/convert.dart';
import 'package:crypto/crypto.dart';
import 'package:path/path.dart' as path;
class CrateHash {
/// Computes a hash uniquely identifying crate content. This takes into account
/// content all all .rs files inside the src directory, as well as Cargo.toml,
/// Cargo.lock, build.rs and cargokit.yaml.
///
/// If [tempStorage] is provided, computed hash is stored in a file in that directory
/// and reused on subsequent calls if the crate content hasn't changed.
static String compute(String manifestDir, {String? tempStorage}) {
return CrateHash._(
manifestDir: manifestDir,
tempStorage: tempStorage,
)._compute();
}
CrateHash._({
required this.manifestDir,
required this.tempStorage,
});
String _compute() {
final files = getFiles();
final tempStorage = this.tempStorage;
if (tempStorage != null) {
final quickHash = _computeQuickHash(files);
final quickHashFolder = Directory(path.join(tempStorage, 'crate_hash'));
quickHashFolder.createSync(recursive: true);
final quickHashFile = File(path.join(quickHashFolder.path, quickHash));
if (quickHashFile.existsSync()) {
return quickHashFile.readAsStringSync();
}
final hash = _computeHash(files);
quickHashFile.writeAsStringSync(hash);
return hash;
} else {
return _computeHash(files);
}
}
/// Computes a quick hash based on files stat (without reading contents). This
/// is used to cache the real hash, which is slower to compute since it involves
/// reading every single file.
String _computeQuickHash(List<File> files) {
final output = AccumulatorSink<Digest>();
final input = sha256.startChunkedConversion(output);
final data = ByteData(8);
for (final file in files) {
input.add(utf8.encode(file.path));
final stat = file.statSync();
data.setUint64(0, stat.size);
input.add(data.buffer.asUint8List());
data.setUint64(0, stat.modified.millisecondsSinceEpoch);
input.add(data.buffer.asUint8List());
}
input.close();
return base64Url.encode(output.events.single.bytes);
}
String _computeHash(List<File> files) {
final output = AccumulatorSink<Digest>();
final input = sha256.startChunkedConversion(output);
void addTextFile(File file) {
// text Files are hashed by lines in case we're dealing with github checkout
// that auto-converts line endings.
final splitter = LineSplitter();
if (file.existsSync()) {
final data = file.readAsStringSync();
final lines = splitter.convert(data);
for (final line in lines) {
input.add(utf8.encode(line));
}
}
}
for (final file in files) {
addTextFile(file);
}
input.close();
final res = output.events.single;
// Truncate to 128bits.
final hash = res.bytes.sublist(0, 16);
return hex.encode(hash);
}
List<File> getFiles() {
final src = Directory(path.join(manifestDir, 'src'));
final files = src
.listSync(recursive: true, followLinks: false)
.whereType<File>()
.toList();
files.sortBy((element) => element.path);
void addFile(String relative) {
final file = File(path.join(manifestDir, relative));
if (file.existsSync()) {
files.add(file);
}
}
addFile('Cargo.toml');
addFile('Cargo.lock');
addFile('build.rs');
addFile('cargokit.yaml');
return files;
}
final String manifestDir;
final String? tempStorage;
}

View File

@@ -0,0 +1,68 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
extension on String {
String resolveSymlink() => File(this).resolveSymbolicLinksSync();
}
class Environment {
/// Current build configuration (debug or release).
static String get configuration =>
_getEnv("CARGOKIT_CONFIGURATION").toLowerCase();
static bool get isDebug => configuration == 'debug';
static bool get isRelease => configuration == 'release';
/// Temporary directory where Rust build artifacts are placed.
static String get targetTempDir => _getEnv("CARGOKIT_TARGET_TEMP_DIR");
/// Final output directory where the build artifacts are placed.
static String get outputDir => _getEnvPath('CARGOKIT_OUTPUT_DIR');
/// Path to the crate manifest (containing Cargo.toml).
static String get manifestDir => _getEnvPath('CARGOKIT_MANIFEST_DIR');
/// Directory inside root project. Not necessarily root folder. Symlinks are
/// not resolved on purpose.
static String get rootProjectDir => _getEnv('CARGOKIT_ROOT_PROJECT_DIR');
// Pod
/// Platform name (macosx, iphoneos, iphonesimulator).
static String get darwinPlatformName =>
_getEnv("CARGOKIT_DARWIN_PLATFORM_NAME");
/// List of architectures to build for (arm64, armv7, x86_64).
static List<String> get darwinArchs =>
_getEnv("CARGOKIT_DARWIN_ARCHS").split(' ');
// Gradle
static String get minSdkVersion => _getEnv("CARGOKIT_MIN_SDK_VERSION");
static String get ndkVersion => _getEnv("CARGOKIT_NDK_VERSION");
static String get sdkPath => _getEnvPath("CARGOKIT_SDK_DIR");
static String get javaHome => _getEnvPath("CARGOKIT_JAVA_HOME");
static List<String> get targetPlatforms =>
_getEnv("CARGOKIT_TARGET_PLATFORMS").split(',');
// CMAKE
static String get targetPlatform => _getEnv("CARGOKIT_TARGET_PLATFORM");
static String _getEnv(String key) {
final res = Platform.environment[key];
if (res == null) {
throw Exception("Missing environment variable $key");
}
return res;
}
static String _getEnvPath(String key) {
final res = _getEnv(key);
if (Directory(res).existsSync()) {
return res.resolveSymlink();
} else {
return res;
}
}
}

View File

@@ -0,0 +1,52 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:logging/logging.dart';
const String kSeparator = "--";
const String kDoubleSeparator = "==";
bool _lastMessageWasSeparator = false;
void _log(LogRecord rec) {
final prefix = '${rec.level.name}: ';
final out = rec.level == Level.SEVERE ? stderr : stdout;
if (rec.message == kSeparator) {
if (!_lastMessageWasSeparator) {
out.write(prefix);
out.writeln('-' * 80);
_lastMessageWasSeparator = true;
}
return;
} else if (rec.message == kDoubleSeparator) {
out.write(prefix);
out.writeln('=' * 80);
_lastMessageWasSeparator = true;
return;
}
out.write(prefix);
out.writeln(rec.message);
_lastMessageWasSeparator = false;
}
void initLogging() {
Logger.root.level = Level.INFO;
Logger.root.onRecord.listen((LogRecord rec) {
final lines = rec.message.split('\n');
for (final line in lines) {
if (line.isNotEmpty || lines.length == 1 || line != lines.last) {
_log(LogRecord(
rec.level,
line,
rec.loggerName,
));
}
}
});
}
void enableVerboseLogging() {
Logger.root.level = Level.ALL;
}

View File

@@ -0,0 +1,309 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:collection/collection.dart';
import 'package:ed25519_edwards/ed25519_edwards.dart';
import 'package:hex/hex.dart';
import 'package:logging/logging.dart';
import 'package:path/path.dart' as path;
import 'package:source_span/source_span.dart';
import 'package:yaml/yaml.dart';
import 'builder.dart';
import 'environment.dart';
import 'rustup.dart';
final _log = Logger('options');
/// A class for exceptions that have source span information attached.
class SourceSpanException implements Exception {
// This is a getter so that subclasses can override it.
/// A message describing the exception.
String get message => _message;
final String _message;
// This is a getter so that subclasses can override it.
/// The span associated with this exception.
///
/// This may be `null` if the source location can't be determined.
SourceSpan? get span => _span;
final SourceSpan? _span;
SourceSpanException(this._message, this._span);
/// Returns a string representation of `this`.
///
/// [color] may either be a [String], a [bool], or `null`. If it's a string,
/// it indicates an ANSI terminal color escape that should be used to
/// highlight the span's text. If it's `true`, it indicates that the text
/// should be highlighted using the default color. If it's `false` or `null`,
/// it indicates that the text shouldn't be highlighted.
@override
String toString({Object? color}) {
if (span == null) return message;
return 'Error on ${span!.message(message, color: color)}';
}
}
enum Toolchain {
stable,
beta,
nightly,
}
class CargoBuildOptions {
final Toolchain toolchain;
final List<String> flags;
CargoBuildOptions({
required this.toolchain,
required this.flags,
});
static Toolchain _toolchainFromNode(YamlNode node) {
if (node case YamlScalar(value: String name)) {
final toolchain =
Toolchain.values.firstWhereOrNull((element) => element.name == name);
if (toolchain != null) {
return toolchain;
}
}
throw SourceSpanException(
'Unknown toolchain. Must be one of ${Toolchain.values.map((e) => e.name)}.',
node.span);
}
static CargoBuildOptions parse(YamlNode node) {
if (node is! YamlMap) {
throw SourceSpanException('Cargo options must be a map', node.span);
}
Toolchain toolchain = Toolchain.stable;
List<String> flags = [];
for (final MapEntry(:key, :value) in node.nodes.entries) {
if (key case YamlScalar(value: 'toolchain')) {
toolchain = _toolchainFromNode(value);
} else if (key case YamlScalar(value: 'extra_flags')) {
if (value case YamlList(nodes: List<YamlNode> list)) {
if (list.every((element) {
if (element case YamlScalar(value: String _)) {
return true;
}
return false;
})) {
flags = list.map((e) => e.value as String).toList();
continue;
}
}
throw SourceSpanException(
'Extra flags must be a list of strings', value.span);
} else {
throw SourceSpanException(
'Unknown cargo option type. Must be "toolchain" or "extra_flags".',
key.span);
}
}
return CargoBuildOptions(toolchain: toolchain, flags: flags);
}
}
extension on YamlMap {
/// Map that extracts keys so that we can do map case check on them.
Map<dynamic, YamlNode> get valueMap =>
nodes.map((key, value) => MapEntry(key.value, value));
}
class PrecompiledBinaries {
final String uriPrefix;
final PublicKey publicKey;
PrecompiledBinaries({
required this.uriPrefix,
required this.publicKey,
});
static PublicKey _publicKeyFromHex(String key, SourceSpan? span) {
final bytes = HEX.decode(key);
if (bytes.length != 32) {
throw SourceSpanException(
'Invalid public key. Must be 32 bytes long.', span);
}
return PublicKey(bytes);
}
static PrecompiledBinaries parse(YamlNode node) {
if (node case YamlMap(valueMap: Map<dynamic, YamlNode> map)) {
if (map
case {
'url_prefix': YamlNode urlPrefixNode,
'public_key': YamlNode publicKeyNode,
}) {
final urlPrefix = switch (urlPrefixNode) {
YamlScalar(value: String urlPrefix) => urlPrefix,
_ => throw SourceSpanException(
'Invalid URL prefix value.', urlPrefixNode.span),
};
final publicKey = switch (publicKeyNode) {
YamlScalar(value: String publicKey) =>
_publicKeyFromHex(publicKey, publicKeyNode.span),
_ => throw SourceSpanException(
'Invalid public key value.', publicKeyNode.span),
};
return PrecompiledBinaries(
uriPrefix: urlPrefix,
publicKey: publicKey,
);
}
}
throw SourceSpanException(
'Invalid precompiled binaries value. '
'Expected Map with "url_prefix" and "public_key".',
node.span);
}
}
/// Cargokit options specified for Rust crate.
class CargokitCrateOptions {
CargokitCrateOptions({
this.cargo = const {},
this.precompiledBinaries,
});
final Map<BuildConfiguration, CargoBuildOptions> cargo;
final PrecompiledBinaries? precompiledBinaries;
static CargokitCrateOptions parse(YamlNode node) {
if (node is! YamlMap) {
throw SourceSpanException('Cargokit options must be a map', node.span);
}
final options = <BuildConfiguration, CargoBuildOptions>{};
PrecompiledBinaries? precompiledBinaries;
for (final entry in node.nodes.entries) {
if (entry
case MapEntry(
key: YamlScalar(value: 'cargo'),
value: YamlNode node,
)) {
if (node is! YamlMap) {
throw SourceSpanException('Cargo options must be a map', node.span);
}
for (final MapEntry(:YamlNode key, :value) in node.nodes.entries) {
if (key case YamlScalar(value: String name)) {
final configuration = BuildConfiguration.values
.firstWhereOrNull((element) => element.name == name);
if (configuration != null) {
options[configuration] = CargoBuildOptions.parse(value);
continue;
}
}
throw SourceSpanException(
'Unknown build configuration. Must be one of ${BuildConfiguration.values.map((e) => e.name)}.',
key.span);
}
} else if (entry.key case YamlScalar(value: 'precompiled_binaries')) {
precompiledBinaries = PrecompiledBinaries.parse(entry.value);
} else {
throw SourceSpanException(
'Unknown cargokit option type. Must be "cargo" or "precompiled_binaries".',
entry.key.span);
}
}
return CargokitCrateOptions(
cargo: options,
precompiledBinaries: precompiledBinaries,
);
}
static CargokitCrateOptions load({
required String manifestDir,
}) {
final uri = Uri.file(path.join(manifestDir, "cargokit.yaml"));
final file = File.fromUri(uri);
if (file.existsSync()) {
final contents = loadYamlNode(file.readAsStringSync(), sourceUrl: uri);
return parse(contents);
} else {
return CargokitCrateOptions();
}
}
}
class CargokitUserOptions {
// When Rustup is installed always build locally unless user opts into
// using precompiled binaries.
static bool defaultUsePrecompiledBinaries() {
return Rustup.executablePath() == null;
}
CargokitUserOptions({
required this.usePrecompiledBinaries,
required this.verboseLogging,
});
CargokitUserOptions._()
: usePrecompiledBinaries = defaultUsePrecompiledBinaries(),
verboseLogging = false;
static CargokitUserOptions parse(YamlNode node) {
if (node is! YamlMap) {
throw SourceSpanException('Cargokit options must be a map', node.span);
}
bool usePrecompiledBinaries = defaultUsePrecompiledBinaries();
bool verboseLogging = false;
for (final entry in node.nodes.entries) {
if (entry.key case YamlScalar(value: 'use_precompiled_binaries')) {
if (entry.value case YamlScalar(value: bool value)) {
usePrecompiledBinaries = value;
continue;
}
throw SourceSpanException(
'Invalid value for "use_precompiled_binaries". Must be a boolean.',
entry.value.span);
} else if (entry.key case YamlScalar(value: 'verbose_logging')) {
if (entry.value case YamlScalar(value: bool value)) {
verboseLogging = value;
continue;
}
throw SourceSpanException(
'Invalid value for "verbose_logging". Must be a boolean.',
entry.value.span);
} else {
throw SourceSpanException(
'Unknown cargokit option type. Must be "use_precompiled_binaries" or "verbose_logging".',
entry.key.span);
}
}
return CargokitUserOptions(
usePrecompiledBinaries: usePrecompiledBinaries,
verboseLogging: verboseLogging,
);
}
static CargokitUserOptions load() {
String fileName = "cargokit_options.yaml";
var userProjectDir = Directory(Environment.rootProjectDir);
while (userProjectDir.parent.path != userProjectDir.path) {
final configFile = File(path.join(userProjectDir.path, fileName));
if (configFile.existsSync()) {
final contents = loadYamlNode(
configFile.readAsStringSync(),
sourceUrl: configFile.uri,
);
final res = parse(contents);
if (res.verboseLogging) {
_log.info('Found user options file at ${configFile.path}');
}
return res;
}
userProjectDir = userProjectDir.parent;
}
return CargokitUserOptions._();
}
final bool usePrecompiledBinaries;
final bool verboseLogging;
}

View File

@@ -0,0 +1,202 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:ed25519_edwards/ed25519_edwards.dart';
import 'package:github/github.dart';
import 'package:logging/logging.dart';
import 'package:path/path.dart' as path;
import 'artifacts_provider.dart';
import 'builder.dart';
import 'cargo.dart';
import 'crate_hash.dart';
import 'options.dart';
import 'rustup.dart';
import 'target.dart';
final _log = Logger('precompile_binaries');
class PrecompileBinaries {
PrecompileBinaries({
required this.privateKey,
required this.githubToken,
required this.repositorySlug,
required this.manifestDir,
required this.targets,
this.androidSdkLocation,
this.androidNdkVersion,
this.androidMinSdkVersion,
this.tempDir,
});
final PrivateKey privateKey;
final String githubToken;
final RepositorySlug repositorySlug;
final String manifestDir;
final List<Target> targets;
final String? androidSdkLocation;
final String? androidNdkVersion;
final int? androidMinSdkVersion;
final String? tempDir;
static String fileName(Target target, String name) {
return '${target.rust}_$name';
}
static String signatureFileName(Target target, String name) {
return '${target.rust}_$name.sig';
}
Future<void> run() async {
final crateInfo = CrateInfo.load(manifestDir);
final targets = List.of(this.targets);
if (targets.isEmpty) {
targets.addAll([
...Target.buildableTargets(),
if (androidSdkLocation != null) ...Target.androidTargets(),
]);
}
_log.info('Precompiling binaries for $targets');
final hash = CrateHash.compute(manifestDir);
_log.info('Computed crate hash: $hash');
final String tagName = 'precompiled_$hash';
final github = GitHub(auth: Authentication.withToken(githubToken));
final repo = github.repositories;
final release = await _getOrCreateRelease(
repo: repo,
tagName: tagName,
packageName: crateInfo.packageName,
hash: hash,
);
final tempDir = this.tempDir != null
? Directory(this.tempDir!)
: Directory.systemTemp.createTempSync('precompiled_');
tempDir.createSync(recursive: true);
final crateOptions = CargokitCrateOptions.load(
manifestDir: manifestDir,
);
final buildEnvironment = BuildEnvironment(
configuration: BuildConfiguration.release,
crateOptions: crateOptions,
targetTempDir: tempDir.path,
manifestDir: manifestDir,
crateInfo: crateInfo,
isAndroid: androidSdkLocation != null,
androidSdkPath: androidSdkLocation,
androidNdkVersion: androidNdkVersion,
androidMinSdkVersion: androidMinSdkVersion,
);
final rustup = Rustup();
for (final target in targets) {
final artifactNames = getArtifactNames(
target: target,
libraryName: crateInfo.packageName,
remote: true,
);
if (artifactNames.every((name) {
final fileName = PrecompileBinaries.fileName(target, name);
return (release.assets ?? []).any((e) => e.name == fileName);
})) {
_log.info("All artifacts for $target already exist - skipping");
continue;
}
_log.info('Building for $target');
final builder =
RustBuilder(target: target, environment: buildEnvironment);
builder.prepare(rustup);
final res = await builder.build();
final assets = <CreateReleaseAsset>[];
for (final name in artifactNames) {
final file = File(path.join(res, name));
if (!file.existsSync()) {
throw Exception('Missing artifact: ${file.path}');
}
final data = file.readAsBytesSync();
final create = CreateReleaseAsset(
name: PrecompileBinaries.fileName(target, name),
contentType: "application/octet-stream",
assetData: data,
);
final signature = sign(privateKey, data);
final signatureCreate = CreateReleaseAsset(
name: signatureFileName(target, name),
contentType: "application/octet-stream",
assetData: signature,
);
bool verified = verify(public(privateKey), data, signature);
if (!verified) {
throw Exception('Signature verification failed');
}
assets.add(create);
assets.add(signatureCreate);
}
_log.info('Uploading assets: ${assets.map((e) => e.name)}');
for (final asset in assets) {
// This seems to be failing on CI so do it one by one
int retryCount = 0;
while (true) {
try {
await repo.uploadReleaseAssets(release, [asset]);
break;
} on Exception catch (e) {
if (retryCount == 10) {
rethrow;
}
++retryCount;
_log.shout(
'Upload failed (attempt $retryCount, will retry): ${e.toString()}');
await Future.delayed(Duration(seconds: 2));
}
}
}
}
_log.info('Cleaning up');
tempDir.deleteSync(recursive: true);
}
Future<Release> _getOrCreateRelease({
required RepositoriesService repo,
required String tagName,
required String packageName,
required String hash,
}) async {
Release release;
try {
_log.info('Fetching release $tagName');
release = await repo.getReleaseByTagName(repositorySlug, tagName);
} on ReleaseNotFound {
_log.info('Release not found - creating release $tagName');
release = await repo.createRelease(
repositorySlug,
CreateRelease.from(
tagName: tagName,
name: 'Precompiled binaries ${hash.substring(0, 8)}',
targetCommitish: null,
isDraft: false,
isPrerelease: false,
body: 'Precompiled binaries for crate $packageName, '
'crate hash $hash.',
));
}
return release;
}
}

View File

@@ -0,0 +1,136 @@
/// This is copied from Cargokit (which is the official way to use it currently)
/// Details: https://fzyzcjy.github.io/flutter_rust_bridge/manual/integrate/builtin
import 'dart:io';
import 'package:collection/collection.dart';
import 'package:path/path.dart' as path;
import 'util.dart';
class _Toolchain {
_Toolchain(
this.name,
this.targets,
);
final String name;
final List<String> targets;
}
class Rustup {
List<String>? installedTargets(String toolchain) {
final targets = _installedTargets(toolchain);
return targets != null ? List.unmodifiable(targets) : null;
}
void installToolchain(String toolchain) {
log.info("Installing Rust toolchain: $toolchain");
runCommand("rustup", ['toolchain', 'install', toolchain]);
_installedToolchains
.add(_Toolchain(toolchain, _getInstalledTargets(toolchain)));
}
void installTarget(
String target, {
required String toolchain,
}) {
log.info("Installing Rust target: $target");
runCommand("rustup", [
'target',
'add',
'--toolchain',
toolchain,
target,
]);
_installedTargets(toolchain)?.add(target);
}
final List<_Toolchain> _installedToolchains;
Rustup() : _installedToolchains = _getInstalledToolchains();
List<String>? _installedTargets(String toolchain) => _installedToolchains
.firstWhereOrNull(
(e) => e.name == toolchain || e.name.startsWith('$toolchain-'))
?.targets;
static List<_Toolchain> _getInstalledToolchains() {
String extractToolchainName(String line) {
// ignore (default) after toolchain name
final parts = line.split(' ');
return parts[0];
}
final res = runCommand("rustup", ['toolchain', 'list']);
// To list all non-custom toolchains, we need to filter out lines that
// don't start with "stable", "beta", or "nightly".
Pattern nonCustom = RegExp(r"^(stable|beta|nightly)");
final lines = res.stdout
.toString()
.split('\n')
.where((e) => e.isNotEmpty && e.startsWith(nonCustom))
.map(extractToolchainName)
.toList(growable: true);
return lines
.map(
(name) => _Toolchain(
name,
_getInstalledTargets(name),
),
)
.toList(growable: true);
}
static List<String> _getInstalledTargets(String toolchain) {
final res = runCommand("rustup", [
'target',
'list',
'--toolchain',
toolchain,
'--installed',
]);
final lines = res.stdout
.toString()
.split('\n')
.where((e) => e.isNotEmpty)
.toList(growable: true);
return lines;
}
bool _didInstallRustSrcForNightly = false;
void installRustSrcForNightly() {
if (_didInstallRustSrcForNightly) {
return;
}
// Useful for -Z build-std
runCommand(
"rustup",
['component', 'add', 'rust-src', '--toolchain', 'nightly'],
);
_didInstallRustSrcForNightly = true;
}
static String? executablePath() {
final envPath = Platform.environment['PATH'];
final envPathSeparator = Platform.isWindows ? ';' : ':';
final home = Platform.isWindows
? Platform.environment['USERPROFILE']
: Platform.environment['HOME'];
final paths = [
if (home != null) path.join(home, '.cargo', 'bin'),
if (envPath != null) ...envPath.split(envPathSeparator),
];
for (final p in paths) {
final rustup = Platform.isWindows ? 'rustup.exe' : 'rustup';
final rustupPath = path.join(p, rustup);
if (File(rustupPath).existsSync()) {
return rustupPath;
}
}
return null;
}
}

Some files were not shown because too many files have changed in this diff Show More