From d1b7e960ee29a0a9c33c9978f1e2e07f77f69e3e Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 19 Jul 2023 16:59:33 -0700 Subject: [PATCH 1/4] add builder script to compile protos into Dart --- mediapipe/dart/dart_builder/.gitignore | 3 + mediapipe/dart/dart_builder/CHANGELOG.md | 3 + mediapipe/dart/dart_builder/README.md | 2 + .../dart/dart_builder/analysis_options.yaml | 30 ++ .../dart/dart_builder/bin/dart_builder.dart | 26 ++ mediapipe/dart/dart_builder/lib/commands.dart | 108 +++++ .../dart/dart_builder/lib/dart_builder.dart | 347 +++++++++++++++ mediapipe/dart/dart_builder/pubspec.lock | 413 ++++++++++++++++++ mediapipe/dart/dart_builder/pubspec.yaml | 20 + 9 files changed, 952 insertions(+) create mode 100644 mediapipe/dart/dart_builder/.gitignore create mode 100644 mediapipe/dart/dart_builder/CHANGELOG.md create mode 100644 mediapipe/dart/dart_builder/README.md create mode 100644 mediapipe/dart/dart_builder/analysis_options.yaml create mode 100644 mediapipe/dart/dart_builder/bin/dart_builder.dart create mode 100644 mediapipe/dart/dart_builder/lib/commands.dart create mode 100644 mediapipe/dart/dart_builder/lib/dart_builder.dart create mode 100644 mediapipe/dart/dart_builder/pubspec.lock create mode 100644 mediapipe/dart/dart_builder/pubspec.yaml diff --git a/mediapipe/dart/dart_builder/.gitignore b/mediapipe/dart/dart_builder/.gitignore new file mode 100644 index 000000000..3a8579040 --- /dev/null +++ b/mediapipe/dart/dart_builder/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/mediapipe/dart/dart_builder/CHANGELOG.md b/mediapipe/dart/dart_builder/CHANGELOG.md new file mode 100644 index 000000000..effe43c82 --- /dev/null +++ b/mediapipe/dart/dart_builder/CHANGELOG.md @@ -0,0 +1,3 @@ +## 1.0.0 + +- Initial version. diff --git a/mediapipe/dart/dart_builder/README.md b/mediapipe/dart/dart_builder/README.md new file mode 100644 index 000000000..3816eca3a --- /dev/null +++ b/mediapipe/dart/dart_builder/README.md @@ -0,0 +1,2 @@ +A sample command-line application with an entrypoint in `bin/`, library code +in `lib/`, and example unit test in `test/`. diff --git a/mediapipe/dart/dart_builder/analysis_options.yaml b/mediapipe/dart/dart_builder/analysis_options.yaml new file mode 100644 index 000000000..dee8927aa --- /dev/null +++ b/mediapipe/dart/dart_builder/analysis_options.yaml @@ -0,0 +1,30 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +# For additional information about configuring this file, see +# https://dart.dev/guides/language/analysis-options diff --git a/mediapipe/dart/dart_builder/bin/dart_builder.dart b/mediapipe/dart/dart_builder/bin/dart_builder.dart new file mode 100644 index 000000000..f66f33c38 --- /dev/null +++ b/mediapipe/dart/dart_builder/bin/dart_builder.dart @@ -0,0 +1,26 @@ +import 'package:args/args.dart'; +import 'package:dart_builder/dart_builder.dart' as db; + +final argParser = ArgParser() + ..addOption( + 'protoc', + help: 'Override the path to protoc', + ) + ..addOption( + 'protobuf', + help: 'Supply a path to google/protobuf if it is not ' + 'next to google/mediapipe', + ) + ..addOption('output', abbr: 'o'); + +Future main(List arguments) async { + final results = argParser.parse(arguments); + final options = db.DartProtoBuilderOptions( + protocPath: results['protoc'], + protobufPath: results['protobuf'], + outputPath: results['output'], + ); + + final protoBuilder = db.DartProtoBuilder(options); + await protoBuilder.run(); +} diff --git a/mediapipe/dart/dart_builder/lib/commands.dart b/mediapipe/dart/dart_builder/lib/commands.dart new file mode 100644 index 000000000..e58f89ebe --- /dev/null +++ b/mediapipe/dart/dart_builder/lib/commands.dart @@ -0,0 +1,108 @@ +import 'dart:io' as io; +import 'package:io/ansi.dart' as ansi; + +class Command { + Command( + this.arguments, { + this.stdOutHandler, + this.stdErrHandler, + this.workingDirectory, + bool? debug, + }) : assert(arguments.isNotEmpty), + _debug = debug ?? false; + + final bool _debug; + final String? workingDirectory; + final List arguments; + void Function(String)? stdOutHandler; + void Function(String)? stdErrHandler; + + io.ProcessResult? _result; + + /// Runs a command and scans its stdout contents for a given needle. + /// If the needle is found, the command is considered to have completed + /// successfully. + factory Command.needleInOutput( + List arguments, { + required String needle, + List ifMissing = const [], + bool shouldExitIfMissing = true, + }) { + return Command( + arguments, + stdOutHandler: (String output) { + bool foundNeedle = false; + bool contains(String line) => line.contains(needle); + if (output.split('\n').any(contains)) { + foundNeedle = true; + } + if (!foundNeedle) { + for (String line in ifMissing) { + io.stderr.writeln(ansi.wrapWith(line, [ansi.red])); + } + if (shouldExitIfMissing) { + io.exit(1); + } + } + }, + ); + } + + factory Command.which( + String commandName, { + String? documentationUrl, + bool shouldExitIfMissing = true, + }) { + return Command( + ['which', commandName], + stdOutHandler: (String output) { + if (output.isEmpty) { + io.stderr.writeAll([ + ansi.wrapWith( + '$commandName not installed or not on path\n', + [ansi.red], + ), + if (documentationUrl != null) + ansi.wrapWith( + 'Visit $documentationUrl for installation instructions\n', + [ansi.red], + ), + ]); + if (shouldExitIfMissing) { + io.exit(1); + } + } + }, + ); + } + + factory Command.run( + List command, { + bool? debug, + String? workingDirectory, + }) => + Command( + command, + stdOutHandler: (String output) => + output.trim().isNotEmpty ? io.stdout.writeln(output.trim()) : null, + stdErrHandler: (String output) => + output.trim().isNotEmpty ? io.stderr.writeln(output.trim()) : null, + debug: debug, + workingDirectory: workingDirectory, + ); + + Future run() async { + if (_debug) { + io.stdout.writeln('>>> ${arguments.join(' ')}'); + } + _result = await io.Process.run( + arguments.first, + arguments.sublist(1), + workingDirectory: workingDirectory, + ); + stdOutHandler?.call(_result!.stdout); + stdErrHandler?.call(_result!.stderr); + } + + int? get exitCode => _result?.exitCode; +} diff --git a/mediapipe/dart/dart_builder/lib/dart_builder.dart b/mediapipe/dart/dart_builder/lib/dart_builder.dart new file mode 100644 index 000000000..bd6f219da --- /dev/null +++ b/mediapipe/dart/dart_builder/lib/dart_builder.dart @@ -0,0 +1,347 @@ +import 'dart:async'; +import 'dart:io' as io; +import 'package:io/ansi.dart' as ansi; +import 'package:path/path.dart' as path; +import 'commands.dart'; + +class DartProtoBuilder { + DartProtoBuilder([this.options = const DartProtoBuilderOptions()]) + : repositoryRoot = io.Directory.current.parent.parent.parent { + // Running this command from within bin (aka: `dart dart_builder.dart`) + // throws everything off by one level. This merges that invocation style + // with the expected, which is `dart bin/dart_builder.dart`. + repositoryRoot = repositoryRoot.absolute.path + .endsWith('mediapipe${io.Platform.pathSeparator}mediapipe') + ? repositoryRoot.parent + : repositoryRoot; + + // Assert we correctly calculated the repository root. + if (!io.Directory( + path.join(repositoryRoot.absolute.path, '.git'), + ).existsSync()) { + io.stderr.writeAll( + [ + 'Executed dart_builder.dart from unexpected directory.\n', + 'Try running: `dart bin${io.Platform.pathSeparator}dart_builder.dart`\n' + ], + ); + io.exit(1); + } + + _mediapipeDir = io.Directory( + path.join(repositoryRoot.absolute.path, 'mediapipe'), + ); + + _dartBuilderDirectory = io.Directory( + path.join( + repositoryRoot.absolute.path, 'mediapipe', 'dart', 'dart_builder'), + ); + + _buildDirectory = io.Directory( + path.join( + io.Directory.current.parent.parent.parent.parent.absolute.path, + 'build', + ), + ); + + _outputDirectory = options.outputPath != null + ? io.Directory(options.outputPath!) + : io.Directory( + path.join( + repositoryRoot.absolute.path, + 'mediapipe', + 'dart', + 'mediapipe', + 'lib', + 'generated', + ), + ); + } + + final DartProtoBuilderOptions options; + + io.Directory repositoryRoot; + + // This is the `mediapipe` directory *within* the repository itself, which + // is also named `mediapipe` + io.Directory get mediapipeDir => _mediapipeDir!; + io.Directory? _mediapipeDir; + + io.Directory get buildDirectory => _buildDirectory!; + io.Directory? _buildDirectory; + + io.Directory get outputDirectory => _outputDirectory!; + io.Directory? _outputDirectory; + + io.Directory get dartBuilderDirectory => _dartBuilderDirectory!; + io.Directory? _dartBuilderDirectory; + + /// Location of local copy of git@github.com:protocolbuffers/protobuf.git. + /// + /// This can either be passed in via `options`, or the script will check for + /// the repository as a sibling to where `mediapipe` is checked out. + /// + /// This starts out as `null` but is set by `_confirmProtobufDirectory`, which + /// either successfully locates the repository or exits. + io.Directory get protobufDirectory => _protobufDirectory!; + io.Directory? _protobufDirectory; + + final _protosToCompile = []; + + Future run() async { + await _confirmOutputDirectories(); + await _confirmProtocPlugin(); + await _confirmProtoc(); + await _confirmProtocGenDart(); + await _confirmProtobufDirectory(); + io.stdout.writeln( + ansi.wrapWith( + 'Dependencies installed correctly.', + [ansi.green], + ), + ); + // await _buildProtos(); + await _buildBarrelFiles(); + } + + Future _buildProtos() async { + await _prepareProtos(); + + await for (io.FileSystemEntity entity + in mediapipeDir.list(recursive: true)) { + if (entity is! io.File) continue; + if (entity.path.endsWith('test.proto')) continue; + if (entity.path.contains('tensorflow')) continue; + if (entity.path.contains('testdata')) continue; + if (entity.path.contains('examples')) continue; + if (!entity.path.endsWith('.proto')) continue; + _protosToCompile.add(entity); + } + + // Lastly, MediaPipe's protobufs have 1 dependency on the `Any` protobuf + // from `google/protobuf`. Thus, for the whole thing to compile, we need to + // add just that class. + final outsideRepository = io.Directory(repositoryRoot.parent.absolute.path); + final anyProto = io.File( + path.join( + outsideRepository.absolute.path, + 'protobuf', + 'src', + 'google', + 'protobuf', + 'any.proto', + ), + ); + _protosToCompile.add(anyProto); + _compileProtos(_protosToCompile); + } + + Future _buildBarrelFiles() async { + final generatedOutput = _GeneratedOutput(); + final fileSystemWalk = outputDirectory.list(recursive: true); + + await for (io.FileSystemEntity entity in fileSystemWalk) { + if (entity is io.Directory) { + generatedOutput.add(entity); + } else if (entity is io.File) { + generatedOutput.addFile(entity.parent, entity); + } + } + + int numBarrelFilesGenerated = 0; + final sep = io.Platform.pathSeparator; + for (final gen in generatedOutput.getDirectories()) { + final fileExports = StringBuffer(); + final nestedBarrelExports = StringBuffer(); + + for (final dir in gen.directories) { + final dirName = dir.absolute.path.split(sep).last; + nestedBarrelExports.writeln("export '$dirName/$dirName.dart';"); + } + + for (final file in gen.files) { + final fileName = file.absolute.path.split(sep).last; + + if (!fileName.endsWith('.dart')) continue; + + // The top-level mediapipe.dart barrel file surfaces multiple collisions + // from classes that exist in various libraries and then again within + // `tasks`. To avoid these collisions, we skip generating the top-level + // barrel file. + if (fileName == 'mediapipe.dart') continue; + + // Skip barrel files generated from a previous run + final parentFolderName = file.parent.absolute.path.split(sep).last; + if (fileName.split('.').first == parentFolderName) continue; + + // Finally, add the valid file export + fileExports.writeln("export '$fileName';"); + } + + final hostDirName = gen.dir.absolute.path.split(sep).last; + final barrelFile = io.File( + '${gen.dir.absolute.path}$sep$hostDirName.dart', + ); + // Delete a pre-existing barrel file. + if (barrelFile.existsSync()) { + io.File(barrelFile.absolute.path).deleteSync(); + } + // Write to our new file. + final spacingNewline = nestedBarrelExports.isNotEmpty ? '\n' : ''; + barrelFile.writeAsStringSync( + '${nestedBarrelExports.toString()}$spacingNewline${fileExports.toString()}\n', + ); + numBarrelFilesGenerated++; + } + io.stdout.writeln( + ansi.wrapWith( + 'Generated $numBarrelFilesGenerated barrel files successfully.', + [ansi.green], + ), + ); + } + + String get protocPath => options.protocPath ?? 'protoc'; + + /// Builds a single protobuf definition's Dart file. + Future _compileProtos(List protoFiles) async { + final command = Command.run( + [ + protocPath, + '-I${repositoryRoot.absolute.path}', + '-I$_protobufCompilationImportPath', + '--dart_out=grpc:${outputDirectory.absolute.path}', + ...protoFiles.map((entity) => entity.absolute.path), + ], + workingDirectory: dartBuilderDirectory.absolute.path, + debug: true, + ); + await command.run(); + if (command.exitCode! != 0) { + io.stderr.writeln('Exiting with ${command.exitCode}'); + io.exit(command.exitCode!); + } + } + + String get _protobufCompilationImportPath => io.Directory( + path.join(protobufDirectory.absolute.path, 'src'), + ).absolute.path; + + Future _confirmOutputDirectories() async { + if (!await _buildDirectory!.exists()) { + _buildDirectory!.create(); + } + if (!await _outputDirectory!.exists()) { + _outputDirectory!.create(); + } + } + + Future _prepareProtos() async { + if (!(await outputDirectory.exists())) { + io.stdout.writeln('Creating output directory'); + outputDirectory.create(); + } + } + + Future _confirmProtoc() async => // + Command.which( + protocPath, + documentationUrl: + 'http://google.github.io/proto-lens/installing-protoc.html', + ).run(); + + Future _confirmProtocGenDart() async => // + Command.which( + 'protoc-gen-dart', + documentationUrl: + 'http://google.github.io/proto-lens/installing-protoc.html', + ).run(); + + Future _confirmProtobufDirectory() async { + _protobufDirectory = options.protobufPath != null + ? io.Directory(options.protobufPath!) + : io.Directory( + path.join(repositoryRoot.parent.absolute.path, 'protobuf'), + ); + if (!await _protobufDirectory!.exists()) { + io.stderr.writeAll( + [ + 'Could not find google/protobuf repository. You can clone this from ', + 'https://github.com/protocolbuffers/protobuf and either pass its ' + 'location via the `--protobuf` flag, or by default, clone it in the ' + 'same directory where you cloned `google/mediapipe`.\n' + '\n', + 'Checked for protobuf library at ${_protobufDirectory!.absolute.path}\n', + ], + ); + io.exit(1); + } + } + + Future _confirmProtocPlugin() async => + Command.needleInOutput(['dart', 'pub', 'global', 'list'], + needle: 'protoc_plugin', + ifMissing: [ + 'protoc_plugin does not seem to be installed', + 'Run `dart pub global activate protoc_plugin` to install it', + ]).run(); +} + +class DartProtoBuilderOptions { + const DartProtoBuilderOptions({ + this.protocPath, + this.protobufPath, + this.outputPath, + }); + + /// Location of `protoc`. Defaults to whatever is found on $PATH. + final String? protocPath; + + /// Location of local copy of git@github.com:protocolbuffers/protobuf.git + final String? protobufPath; + + /// Place to put the generated protobufs. + final String? outputPath; +} + +/// Bundles a set of _GeneratedOutputDirectory objects with helpful getters. +class _GeneratedOutput { + final Map generated = {}; + + void add(io.Directory dir) { + generated[dir.absolute.path] = _GeneratedOutputDirectory(dir); + if (contains(dir.parent)) { + generated[dir.parent.absolute.path]!.addChild(dir); + } + } + + bool contains(io.Directory dir) => generated.containsKey(dir.absolute.path); + + _GeneratedOutputDirectory get(io.Directory dir) { + if (!contains(dir)) { + throw Exception('Unexpectedly asked for unknown directory: ' + '${dir.absolute.path}. Are you recursively walking the file system?'); + } + return generated[dir.absolute.path]!; + } + + void addFile(io.Directory dir, io.File file) => get(dir).addFile(file); + + Iterable<_GeneratedOutputDirectory> getDirectories() sync* { + for (final gen in generated.values) { + yield gen; + } + } +} + +/// Tracks which Dart files were generated where for the purposes of adding +/// barrel files. +class _GeneratedOutputDirectory { + _GeneratedOutputDirectory(this.dir); + final io.Directory dir; + final List directories = []; + final List files = []; + + void addChild(io.Directory dir) => directories.add(dir); + void addFile(io.File file) => files.add(file); +} diff --git a/mediapipe/dart/dart_builder/pubspec.lock b/mediapipe/dart/dart_builder/pubspec.lock new file mode 100644 index 000000000..b82c67f44 --- /dev/null +++ b/mediapipe/dart/dart_builder/pubspec.lock @@ -0,0 +1,413 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: "0816708f5fbcacca324d811297153fe3c8e047beb5c6752e12292d2974c17045" + url: "https://pub.dev" + source: hosted + version: "62.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "21862995c9932cd082f89d72ae5f5e2c110d1a0204ad06e4ebaee8307b76b834" + url: "https://pub.dev" + source: hosted + version: "6.0.0" + ansi: + dependency: "direct main" + description: + name: ansi + sha256: ce4857dd957e8e4fcf1c2f409baa5b262a717a6516085d97532621c47ff87c6e + url: "https://pub.dev" + source: hosted + version: "0.2.2" + ansi_codes: + dependency: transitive + description: + name: ansi_codes + sha256: "48aec9524e26ef234b0376b6bcf92edd9b89ad9c8994ddc7c3b52729af553bed" + url: "https://pub.dev" + source: hosted + version: "0.1.1" + args: + dependency: transitive + description: + name: args + sha256: eef6c46b622e0494a36c5a12d10d77fb4e855501a91c1b9ef9339326e58f0596 + url: "https://pub.dev" + source: hosted + version: "2.4.2" + async: + dependency: transitive + description: + name: async + sha256: "947bfcf187f74dbc5e146c9eb9c0f10c9f8b30743e341481c1e2ed3ecc18c20c" + url: "https://pub.dev" + source: hosted + version: "2.11.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "6cfb5af12253eaf2b368f07bacc5a80d1301a071c73360d746b7f2e32d762c66" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + collection: + dependency: transitive + description: + name: collection + sha256: f092b211a4319e98e5ff58223576de6c2803db36221657b46c82574721240687 + url: "https://pub.dev" + source: hosted + version: "1.17.2" + convert: + dependency: transitive + description: + name: convert + sha256: "0f08b14755d163f6e2134cb58222dd25ea2a2ee8a195e53983d57c075324d592" + url: "https://pub.dev" + source: hosted + version: "3.1.1" + coverage: + dependency: transitive + description: + name: coverage + sha256: "2fb815080e44a09b85e0f2ca8a820b15053982b2e714b59267719e8a9ff17097" + url: "https://pub.dev" + source: hosted + version: "1.6.3" + crypto: + dependency: transitive + description: + name: crypto + sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab + url: "https://pub.dev" + source: hosted + version: "3.0.3" + dart_internal: + dependency: transitive + description: + name: dart_internal + sha256: dae3976f383beddcfcd07ad5291a422df2c8c0a8a03c52cda63ac7b4f26e0f4e + url: "https://pub.dev" + source: hosted + version: "0.2.8" + file: + dependency: transitive + description: + name: file + sha256: "5fc22d7c25582e38ad9a8515372cd9a93834027aacf1801cf01164dac0ffa08c" + url: "https://pub.dev" + source: hosted + version: "7.0.0" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: "25517a4deb0c03aa0f32fd12db525856438902d9c16536311e76cdc57b31d7d1" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: "408e3ca148b31c20282ad6f37ebfa6f4bdc8fede5b74bc2f08d9d92b55db3612" + url: "https://pub.dev" + source: hosted + version: "3.2.0" + glob: + dependency: transitive + description: + name: glob + sha256: "0e7014b3b7d4dac1ca4d6114f82bf1782ee86745b9b42a92c9289c23d8a0ab63" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: "97486f20f9c2f7be8f514851703d0119c3596d14ea63227af6f7a481ef2b2f8b" + url: "https://pub.dev" + source: hosted + version: "3.2.1" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "2aa08ce0341cc9b354a498388e30986515406668dbcc4f7c950c3e715496693b" + url: "https://pub.dev" + source: hosted + version: "4.0.2" + io: + dependency: "direct main" + description: + name: io + sha256: "2ec25704aba361659e10e3e5f5d672068d332fc8ac516421d483a11e5cbd061e" + url: "https://pub.dev" + source: hosted + version: "1.0.4" + js: + dependency: transitive + description: + name: js + sha256: f2c445dce49627136094980615a031419f7f3eb393237e4ecd97ac15dea343f3 + url: "https://pub.dev" + source: hosted + version: "0.6.7" + lints: + dependency: "direct dev" + description: + name: lints + sha256: "0a217c6c989d21039f1498c3ed9f3ed71b354e69873f13a8dfc3c9fe76f1b452" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + logging: + dependency: transitive + description: + name: logging + sha256: "623a88c9594aa774443aa3eb2d41807a48486b5613e67599fb4c41c0ad47c340" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: "1803e76e6653768d64ed8ff2e1e67bea3ad4b923eb5c56a295c3e634bad5960e" + url: "https://pub.dev" + source: hosted + version: "0.12.16" + meta: + dependency: transitive + description: + name: meta + sha256: "3c74dbf8763d36539f114c799d8a2d87343b5067e9d796ca22b5eb8437090ee3" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + mime: + dependency: transitive + description: + name: mime + sha256: e4ff8e8564c03f255408decd16e7899da1733852a9110a58fe6d1b817684a63e + url: "https://pub.dev" + source: hosted + version: "1.0.4" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + package_config: + dependency: transitive + description: + name: package_config + sha256: "1c5b77ccc91e4823a5af61ee74e6b972db1ef98c2ff5a18d3161c982a55448bd" + url: "https://pub.dev" + source: hosted + version: "2.1.0" + path: + dependency: transitive + description: + name: path + sha256: "8829d8a55c13fc0e37127c29fedf290c102f4e40ae94ada574091fe0ff96c917" + url: "https://pub.dev" + source: hosted + version: "1.8.3" + pool: + dependency: transitive + description: + name: pool + sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a" + url: "https://pub.dev" + source: hosted + version: "1.5.1" + protobuf: + dependency: "direct main" + description: + name: protobuf + sha256: "4034a02b7e231e7e60bff30a8ac13a7347abfdac0798595fae0b90a3f0afe759" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "40d3ab1bbd474c4c2328c91e3a7df8c6dd629b79ece4c4bd04bee496a224fb0c" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + shelf: + dependency: transitive + description: + name: shelf + sha256: ad29c505aee705f41a4d8963641f91ac4cee3c8fad5947e033390a7bd8180fa4 + url: "https://pub.dev" + source: hosted + version: "1.4.1" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: a41d3f53c4adf0f57480578c1d61d90342cd617de7fc8077b1304643c2d85c1e + url: "https://pub.dev" + source: hosted + version: "1.1.2" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "9ca081be41c60190ebcb4766b2486a7d50261db7bd0f5d9615f2d653637a84c1" + url: "https://pub.dev" + source: hosted + version: "1.0.4" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: "84cf769ad83aa6bb61e0aa5a18e53aea683395f196a6f39c4c881fb90ed4f7ae" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "708b3f6b97248e5781f493b765c3337db11c5d2c81c3094f10904bfa8004c703" + url: "https://pub.dev" + source: hosted + version: "0.10.12" + source_span: + dependency: transitive + description: + name: source_span + sha256: "53e943d4206a5e30df338fd4c6e7a077e02254531b138a15aec3bd143c1a8b3c" + url: "https://pub.dev" + source: hosted + version: "1.10.0" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "73713990125a6d93122541237550ee3352a2d84baad52d375a4cad2eb9b7ce0b" + url: "https://pub.dev" + source: hosted + version: "1.11.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: ba2aa5d8cc609d96bbb2899c28934f9e1af5cddbd60a827822ea467161eb54e7 + url: "https://pub.dev" + source: hosted + version: "2.1.2" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "556692adab6cfa87322a115640c11f13cb77b3f076ddcc5d6ae3c20242bedcde" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: a29248a84fbb7c79282b40b8c72a1209db169a2e0542bce341da992fe1bc7e84 + url: "https://pub.dev" + source: hosted + version: "1.2.1" + test: + dependency: "direct dev" + description: + name: test + sha256: "67ec5684c7a19b2aba91d2831f3d305a6fd8e1504629c5818f8d64478abf4f38" + url: "https://pub.dev" + source: hosted + version: "1.24.4" + test_api: + dependency: transitive + description: + name: test_api + sha256: "5c2f730018264d276c20e4f1503fd1308dfbbae39ec8ee63c5236311ac06954b" + url: "https://pub.dev" + source: hosted + version: "0.6.1" + test_core: + dependency: transitive + description: + name: test_core + sha256: "6b753899253c38ca0523bb0eccff3934ec83d011705dae717c61ecf209e333c9" + url: "https://pub.dev" + source: hosted + version: "0.5.4" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: facc8d6582f16042dd49f2463ff1bd6e2c9ef9f3d5da3d9b087e244a7b564b3c + url: "https://pub.dev" + source: hosted + version: "1.3.2" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: b8c67f5fa3897b122cf60fe9ff314f7b0ef71eab25c5f8b771480bc338f48823 + url: "https://pub.dev" + source: hosted + version: "11.7.2" + watcher: + dependency: transitive + description: + name: watcher + sha256: "3d2ad6751b3c16cf07c7fca317a1413b3f26530319181b37e3b9039b84fc01d8" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: d88238e5eac9a42bb43ca4e721edba3c08c6354d4a53063afaa568516217621b + url: "https://pub.dev" + source: hosted + version: "2.4.0" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "67d3a8b6c79e1987d19d848b0892e582dbb0c66c57cc1fef58a177dd2aa2823d" + url: "https://pub.dev" + source: hosted + version: "1.2.0" + yaml: + dependency: transitive + description: + name: yaml + sha256: "75769501ea3489fca56601ff33454fe45507ea3bfb014161abc3b43ae25989d5" + url: "https://pub.dev" + source: hosted + version: "3.1.2" +sdks: + dart: ">=3.1.0-149.0.dev <3.2.0" diff --git a/mediapipe/dart/dart_builder/pubspec.yaml b/mediapipe/dart/dart_builder/pubspec.yaml new file mode 100644 index 000000000..e3180a4fe --- /dev/null +++ b/mediapipe/dart/dart_builder/pubspec.yaml @@ -0,0 +1,20 @@ +name: dart_builder +description: A sample command-line application. +version: 1.0.0 +# repository: https://github.com/my_org/my_repo +environment: + sdk: ^3.1.0-149.0.dev + +# Add regular dependencies here. +dependencies: + ansi: ^0.2.2 + args: ^2.4.2 + ffi: ^2.0.2 + fixnum: ^1.1.0 + io: ^1.0.4 + path: ^1.8.3 + protobuf: ^3.0.0 + +dev_dependencies: + lints: ^2.0.0 + test: ^1.21.0 From 8502ed9806b3877c102a218131449c93a82e5502 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 19 Jul 2023 17:08:54 -0700 Subject: [PATCH 2/4] add beginning of Dart-specific MediaPipe implementation --- mediapipe/dart/mediapipe/.gitignore | 7 ++ mediapipe/dart/mediapipe/CHANGELOG.md | 3 + mediapipe/dart/mediapipe/README.md | 39 +++++++ .../dart/mediapipe/analysis_options.yaml | 30 +++++ .../dart/mediapipe/lib/generated/README.md | 5 + mediapipe/dart/mediapipe/lib/mediapipe.dart | 3 + .../lib/src/tasks/core/base_options.dart | 83 ++++++++++++++ .../mediapipe/lib/src/tasks/core/core.dart | 3 + .../lib/src/tasks/core/task_info.dart | 91 +++++++++++++++ .../lib/src/tasks/core/task_runner.dart | 17 +++ .../dart/mediapipe/lib/src/tasks/tasks.dart | 2 + .../mediapipe/lib/src/tasks/text/text.dart | 1 + .../lib/src/tasks/text/text_classifier.dart | 108 ++++++++++++++++++ mediapipe/dart/mediapipe/pubspec.yaml | 16 +++ 14 files changed, 408 insertions(+) create mode 100644 mediapipe/dart/mediapipe/.gitignore create mode 100644 mediapipe/dart/mediapipe/CHANGELOG.md create mode 100644 mediapipe/dart/mediapipe/README.md create mode 100644 mediapipe/dart/mediapipe/analysis_options.yaml create mode 100644 mediapipe/dart/mediapipe/lib/generated/README.md create mode 100644 mediapipe/dart/mediapipe/lib/mediapipe.dart create mode 100644 mediapipe/dart/mediapipe/lib/src/tasks/core/base_options.dart create mode 100644 mediapipe/dart/mediapipe/lib/src/tasks/core/core.dart create mode 100644 mediapipe/dart/mediapipe/lib/src/tasks/core/task_info.dart create mode 100644 mediapipe/dart/mediapipe/lib/src/tasks/core/task_runner.dart create mode 100644 mediapipe/dart/mediapipe/lib/src/tasks/tasks.dart create mode 100644 mediapipe/dart/mediapipe/lib/src/tasks/text/text.dart create mode 100644 mediapipe/dart/mediapipe/lib/src/tasks/text/text_classifier.dart create mode 100644 mediapipe/dart/mediapipe/pubspec.yaml diff --git a/mediapipe/dart/mediapipe/.gitignore b/mediapipe/dart/mediapipe/.gitignore new file mode 100644 index 000000000..3cceda557 --- /dev/null +++ b/mediapipe/dart/mediapipe/.gitignore @@ -0,0 +1,7 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ + +# Avoid committing pubspec.lock for library packages; see +# https://dart.dev/guides/libraries/private-files#pubspeclock. +pubspec.lock diff --git a/mediapipe/dart/mediapipe/CHANGELOG.md b/mediapipe/dart/mediapipe/CHANGELOG.md new file mode 100644 index 000000000..effe43c82 --- /dev/null +++ b/mediapipe/dart/mediapipe/CHANGELOG.md @@ -0,0 +1,3 @@ +## 1.0.0 + +- Initial version. diff --git a/mediapipe/dart/mediapipe/README.md b/mediapipe/dart/mediapipe/README.md new file mode 100644 index 000000000..8b55e735b --- /dev/null +++ b/mediapipe/dart/mediapipe/README.md @@ -0,0 +1,39 @@ + + +TODO: Put a short description of the package here that helps potential users +know whether this package might be useful for them. + +## Features + +TODO: List what your package can do. Maybe include images, gifs, or videos. + +## Getting started + +TODO: List prerequisites and provide or point to information on how to +start using the package. + +## Usage + +TODO: Include short and useful examples for package users. Add longer examples +to `/example` folder. + +```dart +const like = 'sample'; +``` + +## Additional information + +TODO: Tell users more about the package: where to find more information, how to +contribute to the package, how to file issues, what response they can expect +from the package authors, and more. diff --git a/mediapipe/dart/mediapipe/analysis_options.yaml b/mediapipe/dart/mediapipe/analysis_options.yaml new file mode 100644 index 000000000..dee8927aa --- /dev/null +++ b/mediapipe/dart/mediapipe/analysis_options.yaml @@ -0,0 +1,30 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +# For additional information about configuring this file, see +# https://dart.dev/guides/language/analysis-options diff --git a/mediapipe/dart/mediapipe/lib/generated/README.md b/mediapipe/dart/mediapipe/lib/generated/README.md new file mode 100644 index 000000000..b7e443259 --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/generated/README.md @@ -0,0 +1,5 @@ +To generate protocol buffers for Dart (in this folder), navigate to the `dart_builder` directory and run: + +``` +$ dart bin/dart_builder.dart +``` \ No newline at end of file diff --git a/mediapipe/dart/mediapipe/lib/mediapipe.dart b/mediapipe/dart/mediapipe/lib/mediapipe.dart new file mode 100644 index 000000000..f26aab278 --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/mediapipe.dart @@ -0,0 +1,3 @@ +library mediapipe; + +export 'src/tasks/tasks.dart'; diff --git a/mediapipe/dart/mediapipe/lib/src/tasks/core/base_options.dart b/mediapipe/dart/mediapipe/lib/src/tasks/core/base_options.dart new file mode 100644 index 000000000..c3b4dab67 --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/src/tasks/core/base_options.dart @@ -0,0 +1,83 @@ +import 'dart:io' as io; +import 'dart:typed_data'; +import 'package:path/path.dart' as path; +import 'package:protobuf/protobuf.dart' as $pb; + +import '../../../generated/mediapipe/calculators/calculators.dart'; +import '../../../generated/mediapipe/tasks/tasks.dart' as tasks_pb; + +/// Class to extend in task-specific *Options classes. Funnels the three +/// [BaseOptions] attributes into their own object. +abstract class TaskOptions { + TaskOptions({this.modelAssetBuffer, this.modelAssetPath, this.delegate}) + : baseOptions = BaseOptions( + delegate: delegate, + modelAssetBuffer: modelAssetBuffer, + modelAssetPath: modelAssetPath, + ); + + final Uint8List? modelAssetBuffer; + final String? modelAssetPath; + final Delegate? delegate; + + final BaseOptions baseOptions; + + $pb.GeneratedMessage toProto(); + + /// In proto2 syntax, extensions are unique IDs, suitable for keys in a hash + /// map, which power the Extensions pattern for protos to house arbitrary + /// extended data. + /// + /// In proto3, this pattern is replaced with the [Any] protobuf, as the + /// convention for setting the unique identifiers surpassed the maximum upper + /// bound of 29 bits as allocated in the protobuf spec. + $pb.Extension get ext; +} + +final class BaseOptions { + BaseOptions({this.modelAssetBuffer, this.modelAssetPath, this.delegate}); + + /// The model asset file contents as bytes; + Uint8List? modelAssetBuffer; + + /// Path to the model asset file. + String? modelAssetPath; + + /// Acceleration strategy to use. GPU support is currently limited to + /// Ubuntu platform. + Delegate? delegate; + + /// See also: https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/tasks/python/core/base_options.py;l=63-89;rcl=548857458 + tasks_pb.BaseOptions toProto() { + String? absModelPath = + modelAssetPath != null ? path.absolute(modelAssetPath!) : null; + + if (!io.Platform.isLinux && delegate == Delegate.gpu) { + throw Exception( + 'GPU Delegate is not yet supported for ${io.Platform.operatingSystem}', + ); + } + tasks_pb.Acceleration? acceleration; + if (delegate == Delegate.cpu) { + acceleration = tasks_pb.Acceleration.create() + ..tflite = InferenceCalculatorOptions_Delegate_TfLite.create(); + } + + final modelAsset = tasks_pb.ExternalFile.create(); + if (absModelPath != null) { + modelAsset.fileName = absModelPath; + } + if (modelAssetBuffer != null) { + modelAsset.fileContent = modelAssetBuffer!; + } + + final options = tasks_pb.BaseOptions.create()..modelAsset = modelAsset; + if (acceleration != null) { + options.acceleration = acceleration; + } + return options; + } +} + +/// Hardware location to perform the given task. +enum Delegate { cpu, gpu } diff --git a/mediapipe/dart/mediapipe/lib/src/tasks/core/core.dart b/mediapipe/dart/mediapipe/lib/src/tasks/core/core.dart new file mode 100644 index 000000000..d11009bfc --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/src/tasks/core/core.dart @@ -0,0 +1,3 @@ +export 'base_options.dart'; +export 'task_info.dart'; +export 'task_runner.dart'; diff --git a/mediapipe/dart/mediapipe/lib/src/tasks/core/task_info.dart b/mediapipe/dart/mediapipe/lib/src/tasks/core/task_info.dart new file mode 100644 index 000000000..4f016fb18 --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/src/tasks/core/task_info.dart @@ -0,0 +1,91 @@ +import 'package:mediapipe/generated/mediapipe/calculators/calculators.dart'; + +import '../../../generated/mediapipe/framework/framework.dart'; +import '../tasks.dart'; + +class TaskInfo { + TaskInfo({ + required this.taskGraph, + required this.inputStreams, + required this.outputStreams, + required this.taskOptions, + }); + String taskGraph; + List inputStreams; + List outputStreams; + T taskOptions; + + CalculatorGraphConfig generateGraphConfig({ + bool enableFlowLimiting = false, + }) { + assert(inputStreams.isNotEmpty, 'TaskInfo.inputStreams must be non-empty'); + assert( + outputStreams.isNotEmpty, + 'TaskInfo.outputStreams must be non-empty', + ); + + FlowLimiterCalculatorOptions.ext; + final taskSubgraphOptions = CalculatorOptions(); + taskSubgraphOptions.addExtension(taskOptions.ext, taskOptions.toProto()); + + if (!enableFlowLimiting) { + return CalculatorGraphConfig.create() + ..node.add( + CalculatorGraphConfig_Node.create() + ..calculator = taskGraph + ..inputStream.addAll(inputStreams) + ..outputStream.addAll(outputStreams) + ..options = taskSubgraphOptions, + ); + } + + // When a FlowLimiterCalculator is inserted to lower the overall graph + // latency, the task doesn't guarantee that each input must have the + // corresponding output. + final taskSubgraphInputs = + inputStreams.map(_addStreamNamePrefix).toList(); + String finishedStream = 'FINISHED: ${_stripTagIndex(outputStreams.first)}'; + + final flowLimiterOptions = CalculatorOptions.create(); + flowLimiterOptions.setExtension( + FlowLimiterCalculatorOptions.ext, + FlowLimiterCalculatorOptions.create() + ..maxInFlight = 1 + ..maxInQueue = 1, + ); + + final flowLimiter = CalculatorGraphConfig_Node.create() + ..calculator = 'FlowLimiterCalculator' + ..inputStreamInfo.add( + InputStreamInfo.create() + ..tagIndex = 'FINISHED' + ..backEdge = true, + ) + ..inputStream.addAll(inputStreams.map(_stripTagIndex).toList()) + ..inputStream.add(finishedStream) + ..outputStream.addAll( + taskSubgraphInputs.map(_stripTagIndex).toList(), + ) + ..options = flowLimiterOptions; + + final config = CalculatorGraphConfig.create() + ..node.add( + CalculatorGraphConfig_Node.create() + ..calculator = taskGraph + ..inputStream.addAll(taskSubgraphInputs) + ..outputStream.addAll(outputStreams) + ..options = taskSubgraphOptions, + ) + ..node.add(flowLimiter) + ..inputStream.addAll(inputStreams) + ..outputStream.addAll(outputStreams); + return config; + } +} + +String _stripTagIndex(String tagIndexName) => tagIndexName.split(':').last; +String _addStreamNamePrefix(String tagIndexName) { + final split = tagIndexName.split(':'); + split.last = 'trottled_${split.last}'; + return split.join(':'); +} diff --git a/mediapipe/dart/mediapipe/lib/src/tasks/core/task_runner.dart b/mediapipe/dart/mediapipe/lib/src/tasks/core/task_runner.dart new file mode 100644 index 000000000..4809a63f7 --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/src/tasks/core/task_runner.dart @@ -0,0 +1,17 @@ +import '../../../generated/mediapipe/framework/calculator.pb.dart'; + +// TODO: Wrap C++ TaskRunner with this, similarly to this Python wrapper: +// https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/python/framework_bindings.cc?q=python%20framework_bindings.cc +class TaskRunner { + TaskRunner(this.graphConfig); + + final CalculatorGraphConfig graphConfig; + + // TODO: Actually decode this line for correct parameter type: + // https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/tasks/python/text/text_classifier.py;l=181 + Map process(Map data) => {}; +} + +// TODO: Wrap C++ Packet with this, similarly to this Python wrapper: +// https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/python/pybind/packet.h +class Packet {} diff --git a/mediapipe/dart/mediapipe/lib/src/tasks/tasks.dart b/mediapipe/dart/mediapipe/lib/src/tasks/tasks.dart new file mode 100644 index 000000000..1f8266f48 --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/src/tasks/tasks.dart @@ -0,0 +1,2 @@ +export 'core/core.dart'; +export 'text/text.dart'; diff --git a/mediapipe/dart/mediapipe/lib/src/tasks/text/text.dart b/mediapipe/dart/mediapipe/lib/src/tasks/text/text.dart new file mode 100644 index 000000000..86b04eb9a --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/src/tasks/text/text.dart @@ -0,0 +1 @@ +export 'text_classifier.dart'; diff --git a/mediapipe/dart/mediapipe/lib/src/tasks/text/text_classifier.dart b/mediapipe/dart/mediapipe/lib/src/tasks/text/text_classifier.dart new file mode 100644 index 000000000..277919c12 --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/src/tasks/text/text_classifier.dart @@ -0,0 +1,108 @@ +import 'package:protobuf/protobuf.dart' as $pb; +import '../../../generated/mediapipe/tasks/tasks.dart' as tasks_pb; +import '../tasks.dart'; + +class TextClassifier { + /// Primary constructor for [TextClassifier]. + TextClassifier(this.options) + : _taskInfo = TaskInfo( + taskGraph: taskGraphName, + inputStreams: ['$textTag:$textInStreamName'], + outputStreams: [ + '$classificationsTag:$classificationsStreamName' + ], + taskOptions: options, + ) { + _taskRunner = TaskRunner(_taskInfo.generateGraphConfig()); + } + + /// Shortcut constructor which only accepts a local path to the model. + factory TextClassifier.fromAssetPath(String assetPath) => TextClassifier( + TextClassifierOptions(modelAssetPath: assetPath), + ); + + /// Configuration options for this [TextClassifier]. + final TextClassifierOptions options; + final TaskInfo _taskInfo; + + TaskRunner get taskRunner => _taskRunner!; + TaskRunner? _taskRunner; + + static const classificationsStreamName = 'classifications_out'; + static const classificationsTag = 'CLASSIFICATIONS'; + static const textTag = 'TEXT'; + static const textInStreamName = 'text_in'; + static const taskGraphName = + 'mediapipe.tasks.text.text_classifier.TextClassifierGraph'; + + /// Performs classification on the input `text`. + Future classify(String text) async { + // TODO: Actually decode this line to correctly fill up this map parameter + // https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/tasks/python/text/text_classifier.py;l=181 + final outputPackets = taskRunner.process({}); + + // TODO: Obviously this is not real + return tasks_pb.ClassificationResult.create(); + } +} + +class TextClassifierOptions extends TaskOptions { + TextClassifierOptions({ + this.displayNamesLocale, + this.maxResults, + this.scoreThreshold, + this.categoryAllowlist, + this.categoryDenylist, + super.modelAssetBuffer, + super.modelAssetPath, + super.delegate, + }); + + /// The locale to use for display names specified through the TFLite Model + /// Metadata. + String? displayNamesLocale; + + /// The maximum number of top-scored classification results to return. + int? maxResults; + + /// Overrides the ones provided in the model metadata. Results below this + /// value are rejected. + double? scoreThreshold; + + /// Allowlist of category names. If non-empty, classification results whose + /// category name is not in this set will be discarded. Duplicate or unknown + /// category names are ignored. Mutually exclusive with `categoryDenylist`. + List? categoryAllowlist; + + /// Denylist of category names. If non-empty, classification results whose + /// category name is in this set will be discarded. Duplicate or unknown + /// category names are ignored. Mutually exclusive with `categoryAllowList`. + List? categoryDenylist; + + @override + $pb.Extension get ext => tasks_pb.TextClassifierGraphOptions.ext; + + @override + tasks_pb.TextClassifierGraphOptions toProto() { + final classifierOptions = tasks_pb.ClassifierOptions.create(); + if (displayNamesLocale != null) { + classifierOptions.displayNamesLocale = displayNamesLocale!; + } + if (maxResults != null) { + classifierOptions.maxResults = maxResults!; + } + if (scoreThreshold != null) { + classifierOptions.scoreThreshold = scoreThreshold!; + } + if (categoryAllowlist != null) { + classifierOptions.categoryAllowlist.addAll(categoryAllowlist!); + } + if (categoryDenylist != null) { + classifierOptions.categoryDenylist.addAll(categoryDenylist!); + } + + return tasks_pb.TextClassifierGraphOptions.create() + ..baseOptions = baseOptions.toProto() + ..classifierOptions = classifierOptions; + } +} diff --git a/mediapipe/dart/mediapipe/pubspec.yaml b/mediapipe/dart/mediapipe/pubspec.yaml new file mode 100644 index 000000000..8c96f711f --- /dev/null +++ b/mediapipe/dart/mediapipe/pubspec.yaml @@ -0,0 +1,16 @@ +name: mediapipe +description: Flutter plugin for Google's MediaPipe. +version: 1.0.0 +# repository: https://github.com/my_org/my_repo +environment: + sdk: ^3.1.0-149.0.dev + +# Add regular dependencies here. +dependencies: + fixnum: ^1.1.0 + path: ^1.8.3 + protobuf: ^3.0.0 + +dev_dependencies: + lints: ^2.0.0 + test: ^1.21.0 From a53cc096050cab12940651799e13bee21bdcd1dc Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Mon, 24 Jul 2023 11:15:48 -0700 Subject: [PATCH 3/4] add c++ binding chicken scratches --- mediapipe/dart/mediapipe/cc/packet.c | 27 ++++++ mediapipe/dart/mediapipe/cc/packet.cc | 108 +++++++++++++++++++++ mediapipe/dart/mediapipe/cc/task_runner.cc | 40 ++++++++ 3 files changed, 175 insertions(+) create mode 100644 mediapipe/dart/mediapipe/cc/packet.c create mode 100644 mediapipe/dart/mediapipe/cc/packet.cc create mode 100644 mediapipe/dart/mediapipe/cc/task_runner.cc diff --git a/mediapipe/dart/mediapipe/cc/packet.c b/mediapipe/dart/mediapipe/cc/packet.c new file mode 100644 index 000000000..e275fdc23 --- /dev/null +++ b/mediapipe/dart/mediapipe/cc/packet.c @@ -0,0 +1,27 @@ +#include "packet.h" + +char *packet_create_string(char *data) +{ + + // another possibility - + // call MediaPipe functionality here instead of + // having another C++ layer? + + return PacketBinding.create_string(data); +} + +// psueodcode option 1 for high level C-wrapper: +// ClassificationResult classify_text(char* data) { +// packets = create_input_packets(data) +// results = classify(packets) +// return results +// } + +// psueodcode option 2 for high level C-wrapper: +// char* classify_text(char* data) { +// packets = create_input_packets(data) +// // could `results` already be a serioalized proto? +// results = classify(packets) +// // then we return the serialized proto straight to Dart? +// return results +// } \ No newline at end of file diff --git a/mediapipe/dart/mediapipe/cc/packet.cc b/mediapipe/dart/mediapipe/cc/packet.cc new file mode 100644 index 000000000..48d2c7d2b --- /dev/null +++ b/mediapipe/dart/mediapipe/cc/packet.cc @@ -0,0 +1,108 @@ +#include "google3/third_party/mediapipe/framework/packet.h"; + +// Copied from: https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/python/pybind/packet_creator.cc;rcl=549487063 + +#ifdef __cplusplus +extern "C" +{ +#endif + + class PacketBinding + { + Packet create_string(std::string &data) + { + // actually call MediaPipe + } + + Packet create_bool(bool data) + { + } + + Packet create_int8(int8 data) + { + } + + Packet create_int16(int16 data) + { + } + + Packet create_int32(int32 data) + { + } + + Packet create_int64(int64 data) + { + } + + Packet create_uint8(uint8 data) + { + } + + Packet create_uint16(uint16 data) + { + } + + Packet create_uint32(uint32 data) + { + } + + Packet create_uint64(uint64 data) + { + } + + Packet create_float(float data) + { + } + + Packet create_double(double data) + { + } + + Packet create_int_array(std::vector &data) + { + } + + Packet create_float_array(std::vector &data) + { + } + + Packet create_int_vector(std::vector &data) + { + } + + Packet create_bool_vector(std::vector &data) + { + } + + Packet create_float_vector(std::vector &data) + { + } + + Packet create_string_vector(std::vector &data) + { + } + + Packet create_image_vector(std::vector &data) + { + } + + Packet create_packet_vector(std::vector &data) + { + } + + Packet create_string_to_packet_map(std::map &data) + { + } + + Packet create_matrix(Eigen::MatrixXf &matrix, bool transpose) + { + } + + Packet create_from_serialized(const bytes &encoding) + { + } + } + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/mediapipe/dart/mediapipe/cc/task_runner.cc b/mediapipe/dart/mediapipe/cc/task_runner.cc new file mode 100644 index 000000000..dd1e479b2 --- /dev/null +++ b/mediapipe/dart/mediapipe/cc/task_runner.cc @@ -0,0 +1,40 @@ +#include "third_party/mediapipe/tasks/cc/core/task_runner.h" + +// Inspired by: https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/tasks/python/core/pybind/task_runner.cc;l=42;rcl=549487063 + +#ifdef __cplusplus +extern "C" +{ +#endif + + TaskRunner task_runner_create(CalculatorGraphConfig graph_config) + { + } + + Map task_runner_process(TaskRunner runner, Map input_packets) + { + } + + void task_runner_send(TaskRunner runner, Map input_packets) + { + } + + void task_runner_send(TaskRunner runner, Map input_packets) + { + } + + void task_runner_close(TaskRunner runner) + { + } + + void task_runner_restart(TaskRunner runner) + { + } + + CalculatorGraphConfig task_runner_get_graph_config(TaskRunner runner) + { + } + +#ifdef __cplusplus +} +#endif \ No newline at end of file From 805e8300f79774667967387eb17ad06f65299638 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Mon, 14 Aug 2023 14:40:11 -0700 Subject: [PATCH 4/4] experimental spike for proto-based plugin --- mediapipe/dart/dart_builder/.gitignore | 1 + mediapipe/dart/dart_builder/bin/cpp_test.dart | 36 ++ .../dart/dart_builder/lib/dart_builder.dart | 19 +- mediapipe/dart/mediapipe/.gitignore | 2 + mediapipe/dart/mediapipe/Makefile | 10 + mediapipe/dart/mediapipe/ffigen.yaml | 8 + .../lib/src/tasks/core/task_runner.dart | 24 +- .../lib/src/tasks/text/text_classifier.dart | 5 +- .../generated/mediapipe_bindings.dart | 420 ++++++++++++++++++ mediapipe/dart/mediapipe/pubspec.yaml | 2 + .../third_party/mediapipe/base_options.h | 25 ++ .../third_party/mediapipe/category.h | 35 ++ .../mediapipe/classification_result.h | 62 +++ .../mediapipe/classifier_options.h | 48 ++ .../third_party/mediapipe/text_classifier.h | 43 ++ 15 files changed, 722 insertions(+), 18 deletions(-) create mode 100644 mediapipe/dart/dart_builder/bin/cpp_test.dart create mode 100644 mediapipe/dart/mediapipe/Makefile create mode 100644 mediapipe/dart/mediapipe/ffigen.yaml create mode 100644 mediapipe/dart/mediapipe/lib/src/third_party/generated/mediapipe_bindings.dart create mode 100644 mediapipe/dart/mediapipe/third_party/mediapipe/base_options.h create mode 100644 mediapipe/dart/mediapipe/third_party/mediapipe/category.h create mode 100644 mediapipe/dart/mediapipe/third_party/mediapipe/classification_result.h create mode 100644 mediapipe/dart/mediapipe/third_party/mediapipe/classifier_options.h create mode 100644 mediapipe/dart/mediapipe/third_party/mediapipe/text_classifier.h diff --git a/mediapipe/dart/dart_builder/.gitignore b/mediapipe/dart/dart_builder/.gitignore index 3a8579040..f5ac5b0ae 100644 --- a/mediapipe/dart/dart_builder/.gitignore +++ b/mediapipe/dart/dart_builder/.gitignore @@ -1,3 +1,4 @@ # https://dart.dev/guides/libraries/private-files # Created by `dart pub` .dart_tool/ +cc/* diff --git a/mediapipe/dart/dart_builder/bin/cpp_test.dart b/mediapipe/dart/dart_builder/bin/cpp_test.dart new file mode 100644 index 000000000..52ac2a718 --- /dev/null +++ b/mediapipe/dart/dart_builder/bin/cpp_test.dart @@ -0,0 +1,36 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi' as ffi; +import 'dart:io' show Platform, Directory; + +import 'package:path/path.dart' as path; + +// FFI signature of the hello_world C function +typedef IncrementFunc = ffi.Int32 Function(ffi.Int32); +typedef Increment = int Function(int); + +void main() { + // Open the dynamic library + var libraryPath = + path.join(Directory.current.absolute.path, 'cpp', 'main.dylib'); + + // if (Platform.isMacOS) { + // libraryPath = + // path.join(Directory.current.path, 'hello_library', 'libhello.dylib'); + // } + + // if (Platform.isWindows) { + // libraryPath = path.join( + // Directory.current.path, 'hello_library', 'Debug', 'hello.dll'); + // } + + final dylib = ffi.DynamicLibrary.open(libraryPath); + + // Look up the C function 'hello_world' + final Increment increment = + dylib.lookup>('increment').asFunction(); + // Call the function + print(increment(99)); +} diff --git a/mediapipe/dart/dart_builder/lib/dart_builder.dart b/mediapipe/dart/dart_builder/lib/dart_builder.dart index bd6f219da..328b2c832 100644 --- a/mediapipe/dart/dart_builder/lib/dart_builder.dart +++ b/mediapipe/dart/dart_builder/lib/dart_builder.dart @@ -37,13 +37,6 @@ class DartProtoBuilder { repositoryRoot.absolute.path, 'mediapipe', 'dart', 'dart_builder'), ); - _buildDirectory = io.Directory( - path.join( - io.Directory.current.parent.parent.parent.parent.absolute.path, - 'build', - ), - ); - _outputDirectory = options.outputPath != null ? io.Directory(options.outputPath!) : io.Directory( @@ -67,12 +60,11 @@ class DartProtoBuilder { io.Directory get mediapipeDir => _mediapipeDir!; io.Directory? _mediapipeDir; - io.Directory get buildDirectory => _buildDirectory!; - io.Directory? _buildDirectory; - + /// Directory to place compiled protobufs. io.Directory get outputDirectory => _outputDirectory!; io.Directory? _outputDirectory; + /// Location of this command. io.Directory get dartBuilderDirectory => _dartBuilderDirectory!; io.Directory? _dartBuilderDirectory; @@ -100,7 +92,7 @@ class DartProtoBuilder { [ansi.green], ), ); - // await _buildProtos(); + await _buildProtos(); await _buildBarrelFiles(); } @@ -228,16 +220,13 @@ class DartProtoBuilder { ).absolute.path; Future _confirmOutputDirectories() async { - if (!await _buildDirectory!.exists()) { - _buildDirectory!.create(); - } if (!await _outputDirectory!.exists()) { _outputDirectory!.create(); } } Future _prepareProtos() async { - if (!(await outputDirectory.exists())) { + if (!await outputDirectory.exists()) { io.stdout.writeln('Creating output directory'); outputDirectory.create(); } diff --git a/mediapipe/dart/mediapipe/.gitignore b/mediapipe/dart/mediapipe/.gitignore index 3cceda557..871a15102 100644 --- a/mediapipe/dart/mediapipe/.gitignore +++ b/mediapipe/dart/mediapipe/.gitignore @@ -1,6 +1,8 @@ # https://dart.dev/guides/libraries/private-files # Created by `dart pub` .dart_tool/ +lib/generated/google/* +lib/generated/mediapipe/* # Avoid committing pubspec.lock for library packages; see # https://dart.dev/guides/libraries/private-files#pubspeclock. diff --git a/mediapipe/dart/mediapipe/Makefile b/mediapipe/dart/mediapipe/Makefile new file mode 100644 index 000000000..02bc1af0f --- /dev/null +++ b/mediapipe/dart/mediapipe/Makefile @@ -0,0 +1,10 @@ +ffigen: + dart run ffigen --config ffigen.yaml + +compile: + gcc c/text_classifier.c -o c/text_classifier + cd c && gcc -static -c -fPIC *.c -o text_classifier.o + cd c && gcc -shared -o text_classifier.dylib text_classifier.o + +run: + cd c && dart text_classifier_c.dart \ No newline at end of file diff --git a/mediapipe/dart/mediapipe/ffigen.yaml b/mediapipe/dart/mediapipe/ffigen.yaml new file mode 100644 index 000000000..96f1a21cf --- /dev/null +++ b/mediapipe/dart/mediapipe/ffigen.yaml @@ -0,0 +1,8 @@ +name: flutter_mediapipe +description: MediaPipe bindings. + +output: "lib/src/third_party/generated/mediapipe_bindings.dart" +headers: + entry-points: + - "third_party/mediapipe/classification_result.h" + - "third_party/mediapipe/text_classifier.h" diff --git a/mediapipe/dart/mediapipe/lib/src/tasks/core/task_runner.dart b/mediapipe/dart/mediapipe/lib/src/tasks/core/task_runner.dart index 4809a63f7..ca890bbd5 100644 --- a/mediapipe/dart/mediapipe/lib/src/tasks/core/task_runner.dart +++ b/mediapipe/dart/mediapipe/lib/src/tasks/core/task_runner.dart @@ -1,15 +1,35 @@ +import 'dart:ffi' as ffi; +// TODO: This will require a web-specific solution. +import 'dart:io'; +import 'package:path/path.dart' as path; + import '../../../generated/mediapipe/framework/calculator.pb.dart'; +// TODO: Figure out ffi type for Maps +typedef ProcessCC = Map Function(Map data); +typedef Process = Map Function(Map data); + // TODO: Wrap C++ TaskRunner with this, similarly to this Python wrapper: // https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/python/framework_bindings.cc?q=python%20framework_bindings.cc class TaskRunner { - TaskRunner(this.graphConfig); + TaskRunner(this.graphConfig) { + var libraryPath = + path.join(Directory.current.absolute.path, 'cc', 'main.dylib'); + mediaPipe = ffi.DynamicLibrary.open(libraryPath); + } final CalculatorGraphConfig graphConfig; + late ffi.DynamicLibrary mediaPipe; + // TODO: Actually decode this line for correct parameter type: // https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/tasks/python/text/text_classifier.py;l=181 - Map process(Map data) => {}; + Map process(Map data) { + throw UnimplementedError(); + // final Process ccProcess = + // mediaPipe.lookup>('process').asFunction(); + // return ccProcess(data); + } } // TODO: Wrap C++ Packet with this, similarly to this Python wrapper: diff --git a/mediapipe/dart/mediapipe/lib/src/tasks/text/text_classifier.dart b/mediapipe/dart/mediapipe/lib/src/tasks/text/text_classifier.dart index 277919c12..a3821100a 100644 --- a/mediapipe/dart/mediapipe/lib/src/tasks/text/text_classifier.dart +++ b/mediapipe/dart/mediapipe/lib/src/tasks/text/text_classifier.dart @@ -23,6 +23,8 @@ class TextClassifier { /// Configuration options for this [TextClassifier]. final TextClassifierOptions options; + + /// Configuration object passed to the [TaskRunner]. final TaskInfo _taskInfo; TaskRunner get taskRunner => _taskRunner!; @@ -35,11 +37,12 @@ class TextClassifier { static const taskGraphName = 'mediapipe.tasks.text.text_classifier.TextClassifierGraph'; + // TODO: Don't return protobuf objects. Instead, convert to plain Dart objects. /// Performs classification on the input `text`. Future classify(String text) async { // TODO: Actually decode this line to correctly fill up this map parameter // https://source.corp.google.com/piper///depot/google3/third_party/mediapipe/tasks/python/text/text_classifier.py;l=181 - final outputPackets = taskRunner.process({}); + final outputPackets = taskRunner.process({textInStreamName: Object()}); // TODO: Obviously this is not real return tasks_pb.ClassificationResult.create(); diff --git a/mediapipe/dart/mediapipe/lib/src/third_party/generated/mediapipe_bindings.dart b/mediapipe/dart/mediapipe/lib/src/third_party/generated/mediapipe_bindings.dart new file mode 100644 index 000000000..72a6d55ca --- /dev/null +++ b/mediapipe/dart/mediapipe/lib/src/third_party/generated/mediapipe_bindings.dart @@ -0,0 +1,420 @@ +// AUTO GENERATED FILE, DO NOT EDIT. +// +// Generated by `package:ffigen`. +// ignore_for_file: type=lint +import 'dart:ffi' as ffi; + +/// MediaPipe bindings. +class flutter_mediapipe { + /// Holds the symbol lookup function. + final ffi.Pointer Function(String symbolName) + _lookup; + + /// The symbols are looked up in [dynamicLibrary]. + flutter_mediapipe(ffi.DynamicLibrary dynamicLibrary) + : _lookup = dynamicLibrary.lookup; + + /// The symbols are looked up with [lookup]. + flutter_mediapipe.fromLookup( + ffi.Pointer Function(String symbolName) + lookup) + : _lookup = lookup; + + void text_classifier_create( + TextClassifierOptions options, + ) { + return _text_classifier_create( + options, + ); + } + + late final _text_classifier_createPtr = + _lookup>( + 'text_classifier_create'); + late final _text_classifier_create = _text_classifier_createPtr + .asFunction(); + + ffi.Pointer text_classifier_classify( + ffi.Pointer classifier, + ffi.Pointer utf8_text, + ) { + return _text_classifier_classify( + classifier, + utf8_text, + ); + } + + late final _text_classifier_classifyPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('text_classifier_classify'); + late final _text_classifier_classify = + _text_classifier_classifyPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); + + ffi.Pointer text_classifier_classify_simple( + ffi.Pointer utf8_text, + ) { + return _text_classifier_classify_simple( + utf8_text, + ); + } + + late final _text_classifier_classify_simplePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer)>>('text_classifier_classify_simple'); + late final _text_classifier_classify_simple = + _text_classifier_classify_simplePtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); + + void text_classifier_close( + ffi.Pointer classifier, + ) { + return _text_classifier_close( + classifier, + ); + } + + late final _text_classifier_closePtr = + _lookup)>>( + 'text_classifier_close'); + late final _text_classifier_close = _text_classifier_closePtr + .asFunction)>(); +} + +final class __mbstate_t extends ffi.Union { + @ffi.Array.multi([128]) + external ffi.Array __mbstate8; + + @ffi.LongLong() + external int _mbstateL; +} + +final class __darwin_pthread_handler_rec extends ffi.Struct { + external ffi + .Pointer)>> + __routine; + + external ffi.Pointer __arg; + + external ffi.Pointer<__darwin_pthread_handler_rec> __next; +} + +final class _opaque_pthread_attr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_cond_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([40]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_condattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutex_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutexattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_once_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlock_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([192]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlockattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([16]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + external ffi.Pointer<__darwin_pthread_handler_rec> __cleanup_stack; + + @ffi.Array.multi([8176]) + external ffi.Array __opaque; +} + +final class Category extends ffi.Struct { + @ffi.Int() + external int index; + + @ffi.Float() + external double score; + + external ffi.Pointer category_name; + + external ffi.Pointer display_name; +} + +final class Classifications extends ffi.Struct { + external ffi.Pointer categories; + + @ffi.Uint32() + external int categories_count; + + @ffi.Int() + external int head_index; + + external ffi.Pointer head_name; +} + +final class ClassificationResult extends ffi.Struct { + external ffi.Pointer classifications; + + @ffi.Uint32() + external int classifications_count; + + @ffi.Int64() + external int timestamp_ms; + + @ffi.Bool() + external bool has_timestamp_ms; +} + +final class BaseOptions extends ffi.Struct { + external ffi.Pointer model_asset_buffer; + + external ffi.Pointer model_asset_path; +} + +final class ClassifierOptions extends ffi.Struct { + external ffi.Pointer display_names_locale; + + @ffi.Int() + external int max_results; + + @ffi.Float() + external double score_threshold; + + external ffi.Pointer> category_allowlist; + + @ffi.Uint32() + external int category_allowlist_count; + + external ffi.Pointer> category_denylist; + + @ffi.Uint32() + external int category_denylist_count; +} + +final class TextClassifierOptions extends ffi.Struct { + external BaseOptions base_options; + + external ClassifierOptions classifier_options; +} + +typedef TextClassifierResult = ClassificationResult; + +const int true1 = 1; + +const int false1 = 0; + +const int __bool_true_false_are_defined = 1; + +const int __WORDSIZE = 64; + +const int __DARWIN_ONLY_64_BIT_INO_T = 1; + +const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; + +const int __DARWIN_ONLY_VERS_1050 = 1; + +const int __DARWIN_UNIX03 = 1; + +const int __DARWIN_64_BIT_INO_T = 1; + +const int __DARWIN_VERS_1050 = 1; + +const int __DARWIN_NON_CANCELABLE = 0; + +const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; + +const int __DARWIN_C_ANSI = 4096; + +const int __DARWIN_C_FULL = 900000; + +const int __DARWIN_C_LEVEL = 900000; + +const int __STDC_WANT_LIB_EXT1__ = 1; + +const int __DARWIN_NO_LONG_LONG = 0; + +const int _DARWIN_FEATURE_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; + +const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; + +const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; + +const int __has_ptrcheck = 0; + +const int __DARWIN_NULL = 0; + +const int __PTHREAD_SIZE__ = 8176; + +const int __PTHREAD_ATTR_SIZE__ = 56; + +const int __PTHREAD_MUTEXATTR_SIZE__ = 8; + +const int __PTHREAD_MUTEX_SIZE__ = 56; + +const int __PTHREAD_CONDATTR_SIZE__ = 8; + +const int __PTHREAD_COND_SIZE__ = 40; + +const int __PTHREAD_ONCE_SIZE__ = 8; + +const int __PTHREAD_RWLOCK_SIZE__ = 192; + +const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; + +const int USER_ADDR_NULL = 0; + +const int INT8_MAX = 127; + +const int INT16_MAX = 32767; + +const int INT32_MAX = 2147483647; + +const int INT64_MAX = 9223372036854775807; + +const int INT8_MIN = -128; + +const int INT16_MIN = -32768; + +const int INT32_MIN = -2147483648; + +const int INT64_MIN = -9223372036854775808; + +const int UINT8_MAX = 255; + +const int UINT16_MAX = 65535; + +const int UINT32_MAX = 4294967295; + +const int UINT64_MAX = -1; + +const int INT_LEAST8_MIN = -128; + +const int INT_LEAST16_MIN = -32768; + +const int INT_LEAST32_MIN = -2147483648; + +const int INT_LEAST64_MIN = -9223372036854775808; + +const int INT_LEAST8_MAX = 127; + +const int INT_LEAST16_MAX = 32767; + +const int INT_LEAST32_MAX = 2147483647; + +const int INT_LEAST64_MAX = 9223372036854775807; + +const int UINT_LEAST8_MAX = 255; + +const int UINT_LEAST16_MAX = 65535; + +const int UINT_LEAST32_MAX = 4294967295; + +const int UINT_LEAST64_MAX = -1; + +const int INT_FAST8_MIN = -128; + +const int INT_FAST16_MIN = -32768; + +const int INT_FAST32_MIN = -2147483648; + +const int INT_FAST64_MIN = -9223372036854775808; + +const int INT_FAST8_MAX = 127; + +const int INT_FAST16_MAX = 32767; + +const int INT_FAST32_MAX = 2147483647; + +const int INT_FAST64_MAX = 9223372036854775807; + +const int UINT_FAST8_MAX = 255; + +const int UINT_FAST16_MAX = 65535; + +const int UINT_FAST32_MAX = 4294967295; + +const int UINT_FAST64_MAX = -1; + +const int INTPTR_MAX = 9223372036854775807; + +const int INTPTR_MIN = -9223372036854775808; + +const int UINTPTR_MAX = -1; + +const int INTMAX_MAX = 9223372036854775807; + +const int UINTMAX_MAX = -1; + +const int INTMAX_MIN = -9223372036854775808; + +const int PTRDIFF_MIN = -9223372036854775808; + +const int PTRDIFF_MAX = 9223372036854775807; + +const int SIZE_MAX = -1; + +const int RSIZE_MAX = 9223372036854775807; + +const int WCHAR_MAX = 2147483647; + +const int WCHAR_MIN = -2147483648; + +const int WINT_MIN = -2147483648; + +const int WINT_MAX = 2147483647; + +const int SIG_ATOMIC_MIN = -2147483648; + +const int SIG_ATOMIC_MAX = 2147483647; diff --git a/mediapipe/dart/mediapipe/pubspec.yaml b/mediapipe/dart/mediapipe/pubspec.yaml index 8c96f711f..938d7a11f 100644 --- a/mediapipe/dart/mediapipe/pubspec.yaml +++ b/mediapipe/dart/mediapipe/pubspec.yaml @@ -7,10 +7,12 @@ environment: # Add regular dependencies here. dependencies: + ffi: ^2.0.2 fixnum: ^1.1.0 path: ^1.8.3 protobuf: ^3.0.0 dev_dependencies: + ffigen: ^9.0.1 lints: ^2.0.0 test: ^1.21.0 diff --git a/mediapipe/dart/mediapipe/third_party/mediapipe/base_options.h b/mediapipe/dart/mediapipe/third_party/mediapipe/base_options.h new file mode 100644 index 000000000..7adf04503 --- /dev/null +++ b/mediapipe/dart/mediapipe/third_party/mediapipe/base_options.h @@ -0,0 +1,25 @@ +/* Copyright 2023 The MediaPipe Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ +#define THIRD_PARTY_MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ + +// Base options for MediaPipe C Tasks. +struct BaseOptions +{ + // The model asset file contents as a string. + char *model_asset_buffer; + + // The path to the model asset to open and mmap in memory. + char *model_asset_path; +}; +#endif // THIRD_PARTY_MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ diff --git a/mediapipe/dart/mediapipe/third_party/mediapipe/category.h b/mediapipe/dart/mediapipe/third_party/mediapipe/category.h new file mode 100644 index 000000000..550b3c5c0 --- /dev/null +++ b/mediapipe/dart/mediapipe/third_party/mediapipe/category.h @@ -0,0 +1,35 @@ + +/* Copyright 2023 The MediaPipe Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef THIRD_PARTY_MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ +#define THIRD_PARTY_MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ +// Defines a single classification result. +// +// The label maps packed into the TFLite Model Metadata [1] are used to populate +// the 'category_name' and 'display_name' fields. +// +// [1]: https://www.tensorflow.org/lite/convert/metadata +struct Category { + // The index of the category in the classification model output. + int index; + // The score for this category, e.g. (but not necessarily) a probability in + // [0,1]. + float score; + // The optional ID for the category, read from the label map packed in the + // TFLite Model Metadata if present. Not necessarily human-readable. + char *category_name; + // The optional human-readable name for the category, read from the label map + // packed in the TFLite Model Metadata if present. + char *display_name; +}; + +#endif // THIRD_PARTY_MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ diff --git a/mediapipe/dart/mediapipe/third_party/mediapipe/classification_result.h b/mediapipe/dart/mediapipe/third_party/mediapipe/classification_result.h new file mode 100644 index 000000000..472125f7e --- /dev/null +++ b/mediapipe/dart/mediapipe/third_party/mediapipe/classification_result.h @@ -0,0 +1,62 @@ +/* Copyright 2023 The MediaPipe Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ +#define THIRD_PARTY_MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ +#include +#include +#include "category.h" + +// Defines classification results for a given classifier head. +struct Classifications +{ + // The array of predicted categories, usually sorted by descending scores, + // e.g. from high to low probability. + struct Category *categories; + + // The number of elements in the categories array. + uint32_t categories_count; + + // The index of the classifier head (i.e. output tensor) these categories + // refer to. This is useful for multi-head models. + int head_index; + + // The optional name of the classifier head, as provided in the TFLite Model + // Metadata [1] if present. This is useful for multi-head models. + // + // [1]: https://www.tensorflow.org/lite/convert/metadata + char *head_name; +}; + +// Defines classification results of a model. +struct ClassificationResult +{ + // The classification results for each head of the model. + struct Classifications *classifications; + + // The number of classifications in the classifications array. + uint32_t classifications_count; + + // The optional timestamp (in milliseconds) of the start of the chunk of data + // corresponding to these results. + // + // This is only used for classification on time series (e.g. audio + // classification). In these use cases, the amount of data to process might + // exceed the maximum size that the model can process: to solve this, the + // input data is split into multiple chunks starting at different timestamps. + int64_t timestamp_ms; + + // Specifies whether the timestamp contains a valid value. + bool has_timestamp_ms; +}; + +#endif // THIRD_PARTY_MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ diff --git a/mediapipe/dart/mediapipe/third_party/mediapipe/classifier_options.h b/mediapipe/dart/mediapipe/third_party/mediapipe/classifier_options.h new file mode 100644 index 000000000..ca1748246 --- /dev/null +++ b/mediapipe/dart/mediapipe/third_party/mediapipe/classifier_options.h @@ -0,0 +1,48 @@ +/* Copyright 2023 The MediaPipe Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ +#define THIRD_PARTY_MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ +#include + +// Classifier options for MediaPipe C classification Tasks. +struct ClassifierOptions { + // The locale to use for display names specified through the TFLite Model + // Metadata, if any. Defaults to English. + char *display_names_locale; + + // The maximum number of top-scored classification results to return. If < 0, + // all available results will be returned. If 0, an invalid argument error is + // returned. + int max_results; + + // Score threshold to override the one provided in the model metadata (if + // any). Results below this value are rejected. + float score_threshold; + + // The allowlist of category names. If non-empty, detection results whose + // category name is not in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_denylist. + char **category_allowlist; + + // The number of elements in the category allowlist. + uint32_t category_allowlist_count; + + // The denylist of category names. If non-empty, detection results whose + // category name is in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_allowlist. + char **category_denylist; + + // The number of elements in the category denylist. + uint32_t category_denylist_count; +}; +#endif // THIRD_PARTY_MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ diff --git a/mediapipe/dart/mediapipe/third_party/mediapipe/text_classifier.h b/mediapipe/dart/mediapipe/third_party/mediapipe/text_classifier.h new file mode 100644 index 000000000..a85f5b611 --- /dev/null +++ b/mediapipe/dart/mediapipe/third_party/mediapipe/text_classifier.h @@ -0,0 +1,43 @@ +/* Copyright 2023 The MediaPipe Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef THIRD_PARTY_MEDIAPIPE_TASKS_C_TEXT_TEXT_CLASSIFIER_TEXT_CLASSIFIER_H_ +#define THIRD_PARTY_MEDIAPIPE_TASKS_C_TEXT_TEXT_CLASSIFIER_TEXT_CLASSIFIER_H_ +#include "base_options.h" +#include "classification_result.h" +#include "classifier_options.h" + +typedef struct ClassificationResult TextClassifierResult; + +// The options for configuring a MediaPipe text classifier task. +struct TextClassifierOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // file with metadata, accelerator options, op resolver, etc. + struct BaseOptions base_options; + // Options for configuring the classifier behavior, such as score threshold, + // number of results, etc. + struct ClassifierOptions classifier_options; +}; + +// void *text_classifier_options_create(); + +// Creates a TextClassifier from the provided `options`. +void *text_classifier_create(struct TextClassifierOptions *options); + +// Performs classification on the input `text`. +TextClassifierResult *text_classifier_classify(void *classifier, + char *utf8_text); + +// Shuts down the TextClassifier when all the work is done. Frees all memory. +void text_classifier_close(void *classifier); +void text_classifier_result_close(TextClassifierResult *result); + +#endif // THIRD_PARTY_MEDIAPIPE_TASKS_C_TEXT_TEXT_CLASSIFIER_TEXT_CLASSIFIER_H_