diff --git a/lib/appstream.dart b/lib/appstream.dart new file mode 100644 index 000000000..843e052e2 --- /dev/null +++ b/lib/appstream.dart @@ -0,0 +1,2 @@ +export 'src/appstream/appstream_service.dart'; +export 'src/appstream/appstream_utils.dart'; diff --git a/lib/src/appstream/appstream_service.dart b/lib/src/appstream/appstream_service.dart new file mode 100644 index 000000000..66b01619f --- /dev/null +++ b/lib/src/appstream/appstream_service.dart @@ -0,0 +1,227 @@ +import 'dart:collection'; + +import 'package:appstream/appstream.dart'; +import 'package:flutter/foundation.dart'; +import 'package:snowball_stemmer/snowball_stemmer.dart'; + +import '/l10n.dart'; +import 'appstream_utils.dart'; + +extension _Tokenizer on String { + List tokenize() => split(RegExp('\\W')); +} + +class _CachedComponent { + final AppstreamComponent component; + final String id; + late String name; + late List keywords; + late List summary; + late List description; + late String origin; + late String package; + late List mediaTypes; + + _CachedComponent(this.component) : id = component.id.toLowerCase() { + name = _getLocalizedComponentAttribute(component.name)?.toLowerCase() ?? ''; + keywords = _getLocalizedComponentAttribute(component.keywords) + ?.map((e) => e.toLowerCase()) + .toList() ?? + []; + summary = _getLocalizedComponentAttribute(component.summary) + ?.toLowerCase() + .tokenize() ?? + []; + description = _getLocalizedComponentAttribute(component.description) + ?.toLowerCase() + .tokenize() ?? + []; + origin = ''; // XXX: https://github.com/canonical/appstream.dart/issues/25 + package = component.package?.toLowerCase() ?? ''; + mediaTypes = []; + for (final provider in component.provides) { + if (provider is AppstreamProvidesMediatype) { + mediaTypes.add(provider.mediaType.toLowerCase()); + } + } + } + + static T? _getLocalizedComponentAttribute(Map attribute) { + final languageKey = bestLanguageKey(attribute.keys); + if (languageKey == null) return null; + return attribute[languageKey]; + } + + int match(List tokens) { + int score = _MatchScore.none.value; + for (final token in tokens) { + if (id.toLowerCase().contains(token)) { + score |= _MatchScore.id.value; + } + if (name.contains(token)) { + score |= _MatchScore.name.value; + } + if (keywords.contains(token)) { + score |= _MatchScore.keyword.value; + } + if (summary.contains(token)) { + score |= _MatchScore.summary.value; + } + if (description.contains(token)) { + score |= _MatchScore.description.value; + } + if (origin.contains(token)) { + score |= _MatchScore.origin.value; + } + if (package.contains(token)) { + score |= _MatchScore.pkgName.value; + } + if (mediaTypes.any((e) => e.contains(token))) { + score |= _MatchScore.mediaType.value; + } + if (score == _MatchScore.all.value) break; + } + return score; + } + + @override + bool operator ==(Object other) => + other is _CachedComponent && component.id == other.component.id; + + @override + int get hashCode => component.id.hashCode; +} + +enum _MatchScore { + none(0), + mediaType(1 << 0), + pkgName(1 << 1), + origin(1 << 2), + description(1 << 3), + summary(1 << 4), + keyword(1 << 5), + name(1 << 6), + id(1 << 7), + all(1 << 0 | 1 << 1 | 1 << 2 | 1 << 3 | 1 << 4 | 1 << 5 | 1 << 6 | 1 << 7); + + final int value; + const _MatchScore(this.value); +} + +class _ScoredComponent { + final int score; + final AppstreamComponent component; + const _ScoredComponent(this.score, this.component); +} + +class AppstreamService { + final AppstreamPool _pool; + late final Future _loader = _pool.load().then((_) => _populateCache()); + + // TODO: cache AppstreamPool + AppstreamService({@visibleForTesting AppstreamPool? pool}) + : _pool = pool ?? AppstreamPool() { + PlatformDispatcher.instance.onLocaleChanged = () async { + await _loader; + _populateCache(); + }; + } + + final HashSet<_CachedComponent> _cache = HashSet<_CachedComponent>(); + @visibleForTesting + int get cacheSize => _cache.length; + + void _populateCache() { + _cache.clear(); + for (final component in _pool.components) { + _cache.add(_CachedComponent(component)); + } + } + + List get _greylist => + lookupAppLocalizations(PlatformDispatcher.instance.locale) + .appstreamSearchGreylist + .split(';'); + + Future init() async => _loader; + + static final stemmersMap = { + 'ar': Algorithm.arabic, + 'hy': Algorithm.armenian, + 'eu': Algorithm.basque, + 'ca': Algorithm.catalan, + 'da': Algorithm.danish, + 'nl': Algorithm.dutch, + 'en': Algorithm.english, + 'fi': Algorithm.finnish, + 'fr': Algorithm.french, + 'de': Algorithm.german, + 'el': Algorithm.greek, + 'hi': Algorithm.hindi, + 'hu': Algorithm.hungarian, + 'id': Algorithm.indonesian, + 'ga': Algorithm.irish, + 'it': Algorithm.italian, + 'lt': Algorithm.lithuanian, + 'ne': Algorithm.nepali, + 'nb': Algorithm.norwegian, + 'pt': Algorithm.portuguese, + 'ro': Algorithm.romanian, + 'ru': Algorithm.russian, + 'sr': Algorithm.serbian, + 'es': Algorithm.spanish, + 'sv': Algorithm.swedish, + 'ta': Algorithm.tamil, + 'tr': Algorithm.turkish, + 'yi': Algorithm.yiddish, + }; + + // Re-implementation of as_pool_build_search_tokens() + // (https://www.freedesktop.org/software/appstream/docs/api/appstream-AsPool.html#as-pool-build-search-tokens) + List _buildSearchTokens(String search) { + final words = search.toLowerCase().split(RegExp(r'\s')); + // Filter out too generic search terms + words.removeWhere((element) => _greylist.contains(element)); + if (words.isEmpty) { + words.addAll(search.toLowerCase().split(RegExp(r'\s'))); + } + // Filter out short tokens, and those containing markup + words.removeWhere( + (element) => element.length <= 1 || element.contains(RegExp(r'[<>()]')), + ); + // Extract only the common stems from the tokens + final algorithm = + stemmersMap[PlatformDispatcher.instance.locale.languageCode]; + if (algorithm != null) { + final stemmer = SnowballStemmer(algorithm); + return words.map((element) => stemmer.stem(element)).toSet().toList(); + } else { + return words; + } + } + + // Re-implementation of as_pool_search() + // (https://www.freedesktop.org/software/appstream/docs/api/appstream-AsPool.html#as-pool-search) + Future> search(String search) async { + final tokens = _buildSearchTokens(search); + await _loader; + if (tokens.isEmpty) { + if (search.length <= 1) { + // Search query too broad, matching everything + return _pool.components; + } else { + // No valid search tokens + return []; + } + } + final scored = <_ScoredComponent>[]; + for (final entry in _cache) { + final score = entry.match(tokens); + if (score > 0) { + scored.add(_ScoredComponent(score, entry.component)); + } + } + scored.sort((a, b) => b.score.compareTo(a.score)); + return scored.map((e) => e.component).toList(); + } +} diff --git a/lib/src/appstream/appstream_utils.dart b/lib/src/appstream/appstream_utils.dart new file mode 100644 index 000000000..eba11de22 --- /dev/null +++ b/lib/src/appstream/appstream_utils.dart @@ -0,0 +1,66 @@ +import 'dart:io'; +import 'dart:ui'; + +import 'package:appstream/appstream.dart'; +import 'package:collection/collection.dart'; + +// TODO: uncomment once we re-add packagekit +// import 'package:packagekit/packagekit.dart'; +// import 'package:ubuntu_service/ubuntu_service.dart'; + +String? bestLanguageKey(Iterable keys, {Locale? locale}) { + locale ??= PlatformDispatcher.instance.locale; + if (locale.toLanguageTag() != 'und') { + var key = '${locale.languageCode}_${locale.countryCode}'; + if (keys.contains(key)) return key; + key = locale.languageCode; + if (keys.contains(key)) return key; + } + const fallback = 'C'; + if (keys.contains(fallback)) return fallback; + return null; +} + +extension LocalizedComponent on AppstreamComponent { + String localizedName({Locale? locale}) => + name[bestLanguageKey(name.keys, locale: locale)] ?? ''; + String localizedSummary({Locale? locale}) => + summary[bestLanguageKey(summary.keys, locale: locale)] ?? ''; + String localizedDescription({Locale? locale}) => + description[bestLanguageKey(description.keys, locale: locale)] ?? ''; +} + +// TODO: uncomment once we re-add packagekit +// extension PackageKitId on AppstreamComponent { +// Future get packageKitId => +// getService().resolve(package ?? id); +// } + +int _sizeComparison(int? a, int? b) => a?.compareTo(b ?? 0) ?? 0; + +extension Icons on AppstreamComponent { + String? get icon { + final cached = icons.whereType().toList(); + cached.sort((a, b) => _sizeComparison(b.width, a.width)); + //for (final icon in cached) { + // XXX: we need the origin to determine if a cached icon exists on disk + // (https://github.com/canonical/appstream.dart/issues/25) + //} + + final local = icons.whereType().toList(); + local.sort((a, b) => _sizeComparison(b.width, a.width)); + for (final icon in local) { + if (File(icon.filename).existsSync()) { + return icon.filename; + } + } + + final stock = icons.whereType().firstOrNull; + if (stock != null) { + // TODO: check whether the stock icon exists on disk, and return it + } + + final remote = icons.whereType().firstOrNull; + return remote?.url; + } +} diff --git a/lib/src/l10n/app_en.arb b/lib/src/l10n/app_en.arb index 757e51f5c..d8a88784a 100644 --- a/lib/src/l10n/app_en.arb +++ b/lib/src/l10n/app_en.arb @@ -1,4 +1,5 @@ { + "appstreamSearchGreylist": "app;application;package;program;programme;suite;tool", "detailPageChannelLabel": "Channel", "detailPageConfinementLabel": "Confinement", "detailPageContactPublisherLabel": "Contact {publisher}", diff --git a/pubspec.yaml b/pubspec.yaml index 06e0800d3..07af04f35 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -8,6 +8,7 @@ environment: flutter: '^3.10.6' dependencies: + appstream: ^0.2.8 args: ^2.4.2 cached_network_image: ^3.2.3 collection: ^1.17.0 @@ -30,6 +31,7 @@ dependencies: shimmer: ^3.0.0 snapcraft_launcher: ^0.1.0 snapd: ^0.4.11 + snowball_stemmer: ^0.1.0 ubuntu_localizations: ^0.3.3 ubuntu_service: ^0.2.2 ubuntu_test: ^0.1.0-0 diff --git a/test/appstream_service_test.dart b/test/appstream_service_test.dart new file mode 100644 index 000000000..b190bdfe2 --- /dev/null +++ b/test/appstream_service_test.dart @@ -0,0 +1,140 @@ +import 'package:app_center/appstream.dart'; +import 'package:appstream/appstream.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:mockito/annotations.dart'; +import 'package:mockito/mockito.dart'; + +import 'appstream_service_test.mocks.dart'; + +@GenerateMocks([AppstreamPool]) +void main() { + late AppstreamPool pool; + late List components; + late AppstreamService service; + + const component1 = AppstreamComponent( + id: 'com.play0ad.zeroad', + type: AppstreamComponentType.desktopApplication, + package: '0ad', + name: {'C': '0 A.D.'}, + summary: {'C': 'Real-Time Strategy Game of Ancient Warfare'}, + description: { + 'C': '''0 A.D. is a real-time strategy (RTS) game of ancient warfare. + Each civilization is complete with substantially unique artwork, + technologies and civilization bonuses.''' + }, + keywords: { + 'C': [ + 'RTS', + 'Real-Time Strategy', + 'Economic Simulation Game', + 'History', + 'Warfare', + ] + }, + provides: [AppstreamProvidesMediatype('application/x-pyromod+zip')], + ); + + const component2 = AppstreamComponent( + id: 'qbrew.desktop', + type: AppstreamComponentType.desktopApplication, + package: 'qbrew', + name: {'C': 'QBrew'}, + summary: {}, + description: {}, + keywords: {}, + ); + + const component3 = AppstreamComponent( + id: 'gperiodic.desktop', + type: AppstreamComponentType.desktopApplication, + package: 'gperiodic', + name: {'C': 'GPeriodic'}, + summary: {}, + description: {}, + keywords: {}, + ); + + setUp(() { + pool = MockAppstreamPool(); + components = []; + when(pool.components).thenReturn(components); + service = AppstreamService(pool: pool); + }); + + test('initialize service', () async { + verifyNever(pool.load()); + await service.init(); + verify(pool.load()).called(1); + expect(service.cacheSize, 0); + }); + + test('load and cache components', () async { + components.add(component1); + await service.init(); + expect(service.cacheSize, 1); + }); + + test('search', () async { + components.addAll([component1, component2, component3]); + await service.init(); + expect(service.cacheSize, 3); + + // Match on package ID + var results = await service.search('play0ad'); + expect(results.length, 1); + expect(results[0], component1); + + // Match on name + results = await service.search('A.D.'); + expect(results.length, 1); + expect(results[0], component1); + + // Match on keywords + results = await service.search('rts'); + expect(results.length, 1); + expect(results[0], component1); + + // Match on summary + results = await service.search('game'); + expect(results.length, 1); + expect(results[0], component1); + + // Match on description + results = await service.search('artwork'); + expect(results.length, 1); + expect(results[0], component1); + + // Match on package name + results = await service.search('0ad'); + expect(results.length, 1); + expect(results[0], component1); + + // Match on media type + results = await service.search('pyromod'); + expect(results.length, 1); + expect(results[0], component1); + + // Match several components + results = await service.search('desktop'); + expect(results.length, 2); + expect(results.contains(component2), isTrue); + expect(results.contains(component3), isTrue); + + // Empty search matches all components + results = await service.search(''); + expect(results.length, 3); + + // Invalid search + results = await service.search(''); + expect(results, isEmpty); + + // 'application' is grey-listed + results = await service.search('foobar application'); + expect(results, isEmpty); + + // 'application' and 'tool' are grey-listed + results = await service.search('package tool'); + expect(results, isEmpty); + }); +} diff --git a/test/appstream_service_test.mocks.dart b/test/appstream_service_test.mocks.dart new file mode 100644 index 000000000..ea3663ef1 --- /dev/null +++ b/test/appstream_service_test.mocks.dart @@ -0,0 +1,45 @@ +// Mocks generated by Mockito 5.4.2 from annotations +// in app_center/test/appstream_service_test.dart. +// Do not manually edit this file. + +// ignore_for_file: no_leading_underscores_for_library_prefixes +import 'dart:async' as _i4; + +import 'package:appstream/src/component.dart' as _i3; +import 'package:appstream/src/pool.dart' as _i2; +import 'package:mockito/mockito.dart' as _i1; + +// ignore_for_file: type=lint +// ignore_for_file: avoid_redundant_argument_values +// ignore_for_file: avoid_setters_without_getters +// ignore_for_file: comment_references +// ignore_for_file: implementation_imports +// ignore_for_file: invalid_use_of_visible_for_testing_member +// ignore_for_file: prefer_const_constructors +// ignore_for_file: unnecessary_parenthesis +// ignore_for_file: camel_case_types +// ignore_for_file: subtype_of_sealed_class + +/// A class which mocks [AppstreamPool]. +/// +/// See the documentation for Mockito's code generation for more information. +class MockAppstreamPool extends _i1.Mock implements _i2.AppstreamPool { + MockAppstreamPool() { + _i1.throwOnMissingStub(this); + } + + @override + List<_i3.AppstreamComponent> get components => (super.noSuchMethod( + Invocation.getter(#components), + returnValue: <_i3.AppstreamComponent>[], + ) as List<_i3.AppstreamComponent>); + @override + _i4.Future load() => (super.noSuchMethod( + Invocation.method( + #load, + [], + ), + returnValue: _i4.Future.value(), + returnValueForMissingStub: _i4.Future.value(), + ) as _i4.Future); +}