Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(vertexai): add Imagen support #16976

Merged
merged 27 commits into from
Feb 20, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
7ce745f
first pass of imagen
cynthiajoan Jan 8, 2025
7088f42
working demo
cynthiajoan Jan 8, 2025
e826339
Separate generateImages and generateImagesGCS
cynthiajoan Jan 9, 2025
adb49f8
expose ImagenImage basetype for future
cynthiajoan Jan 13, 2025
2a5b26c
updates after api review
cynthiajoan Jan 13, 2025
cd59d74
organize example page and functions
cynthiajoan Jan 23, 2025
11e8876
fix analyzer
cynthiajoan Jan 23, 2025
f2dd39a
Merge branch 'vertexai/example_update' into vertexai/imagen
cynthiajoan Jan 24, 2025
8eefb6e
imagen model working with new example layout
cynthiajoan Jan 24, 2025
8922f9b
Finish up all functionality
cynthiajoan Jan 24, 2025
32825dd
Merge branch 'main' into vertexai/imagen
cynthiajoan Jan 24, 2025
9b8fda9
remove the storage uri prompt page after merge
cynthiajoan Jan 24, 2025
7b31d08
Merge branch 'main' into vertexai/imagen
cynthiajoan Feb 3, 2025
d95abfe
Add example for gcs generation
cynthiajoan Feb 4, 2025
74909e5
Add ImagenImagesBlockedException
cynthiajoan Feb 4, 2025
01396d3
Add unit tests
cynthiajoan Feb 4, 2025
232ac9c
update the year
cynthiajoan Feb 4, 2025
9465335
finalize with the name
cynthiajoan Feb 7, 2025
7117d53
Hide gcs api and some review feedback
cynthiajoan Feb 11, 2025
257e2ec
Update packages/firebase_vertexai/firebase_vertexai/lib/src/imagen_ap…
cynthiajoan Feb 11, 2025
fa45ad2
Update packages/firebase_vertexai/firebase_vertexai/lib/src/imagen_mo…
cynthiajoan Feb 11, 2025
75b331e
review comments
cynthiajoan Feb 11, 2025
9b33811
Merge branch 'vertexai/imagen' of https://github.com/firebase/flutter…
cynthiajoan Feb 11, 2025
a3cfe59
More review comment update
cynthiajoan Feb 11, 2025
c8eb4f9
Merge branch 'main' into vertexai/imagen
cynthiajoan Feb 11, 2025
8f21751
fix analyzer
cynthiajoan Feb 18, 2025
3ea7571
Merge branch 'main' into vertexai/imagen
cynthiajoan Feb 18, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import UIKit
import Flutter

@UIApplicationMain
@main
@objc class AppDelegate: FlutterAppDelegate {
override func application(
_ application: UIApplication,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import 'pages/function_calling_page.dart';
import 'pages/image_prompt_page.dart';
import 'pages/token_count_page.dart';
import 'pages/schema_page.dart';
import 'pages/storage_uri_page.dart';
import 'pages/imagen_page.dart';

// REQUIRED if you want to run on Web
const FirebaseOptions? options = null;
Expand Down Expand Up @@ -79,7 +79,7 @@ class _HomeScreenState extends State<HomeScreen> {
title: 'Function Calling',
), // function calling will initial its own model
ImagePromptPage(title: 'Image Prompt', model: widget.model),
StorageUriPromptPage(title: 'Storage URI Prompt', model: widget.model),
ImagenPage(title: 'Imagen Model', model: widget.model),
SchemaPromptPage(title: 'Schema Prompt', model: widget.model),
];

Expand Down Expand Up @@ -134,11 +134,11 @@ class _HomeScreenState extends State<HomeScreen> {
),
BottomNavigationBarItem(
icon: Icon(
Icons.folder,
Icons.image_search,
color: Theme.of(context).colorScheme.primary,
),
label: 'Storage URI Prompt',
tooltip: 'Storage URI Prompt',
label: 'Imagen Model',
tooltip: 'Imagen Model',
),
BottomNavigationBarItem(
icon: Icon(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,14 +89,28 @@ class _ImagePromptPageState extends State<ImagePromptPage> {
const SizedBox.square(
dimension: 15,
),
ElevatedButton(
onPressed: !_loading
? () async {
await _sendImagePrompt(_textController.text);
}
: null,
child: const Text('Send Image Prompt'),
),
if (!_loading)
IconButton(
onPressed: () async {
await _sendImagePrompt(_textController.text);
},
icon: Icon(
Icons.image,
color: Theme.of(context).colorScheme.primary,
),
),
if (!_loading)
IconButton(
onPressed: () async {
await _sendStorageUriPrompt(_textController.text);
},
icon: Icon(
Icons.storage,
color: Theme.of(context).colorScheme.primary,
),
)
else
const CircularProgressIndicator(),
],
),
),
Expand Down Expand Up @@ -162,6 +176,49 @@ class _ImagePromptPageState extends State<ImagePromptPage> {
}
}

Future<void> _sendStorageUriPrompt(String message) async {
setState(() {
_loading = true;
});
try {
final content = [
Content.multi([
TextPart(message),
FileData(
'image/jpeg',
'gs://vertex-ai-example-ef5a2.appspot.com/foodpic.jpg',
),
]),
];
_generatedContent.add(MessageData(text: message, fromUser: true));

var response = await widget.model.generateContent(content);
var text = response.text;
_generatedContent.add(MessageData(text: text, fromUser: false));

if (text == null) {
_showError('No response from API.');
return;
} else {
setState(() {
_loading = false;
_scrollDown();
});
}
} catch (e) {
_showError(e.toString());
setState(() {
_loading = false;
});
} finally {
_textController.clear();
setState(() {
_loading = false;
});
_textFieldFocus.requestFocus();
}
}

void _showError(String message) {
showDialog<void>(
context: context,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import 'package:flutter/material.dart';
import 'package:firebase_vertexai/firebase_vertexai.dart';
//import 'package:firebase_storage/firebase_storage.dart';
import '../widgets/message_widget.dart';

class ImagenPage extends StatefulWidget {
const ImagenPage({
super.key,
required this.title,
required this.model,
});

final String title;
final GenerativeModel model;

@override
State<ImagenPage> createState() => _ImagenPageState();
}

class _ImagenPageState extends State<ImagenPage> {
final ScrollController _scrollController = ScrollController();
final TextEditingController _textController = TextEditingController();
final FocusNode _textFieldFocus = FocusNode();
final List<MessageData> _generatedContent = <MessageData>[];
bool _loading = false;
late final ImagenModel _imagenModel;

@override
void initState() {
super.initState();
var generationConfig = ImagenGenerationConfig(
negativePrompt: 'frog',
numberOfImages: 1,
aspectRatio: ImagenAspectRatio.square1x1,
imageFormat: ImagenFormat.jpeg(compressionQuality: 75),
);
_imagenModel = FirebaseVertexAI.instance.imagenModel(
model: 'imagen-3.0-generate-001',
generationConfig: generationConfig,
safetySettings: ImagenSafetySettings(
ImagenSafetyFilterLevel.blockLowAndAbove,
ImagenPersonFilterLevel.allowAdult,
),
);
}

void _scrollDown() {
WidgetsBinding.instance.addPostFrameCallback(
(_) => _scrollController.animateTo(
_scrollController.position.maxScrollExtent,
duration: const Duration(
milliseconds: 750,
),
curve: Curves.easeOutCirc,
),
);
}

@override
Widget build(BuildContext context) {
return Scaffold(
appBar: AppBar(
title: Text(widget.title),
),
body: Padding(
padding: const EdgeInsets.all(8),
child: Column(
mainAxisAlignment: MainAxisAlignment.center,
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Expanded(
child: ListView.builder(
controller: _scrollController,
itemBuilder: (context, idx) {
return MessageWidget(
text: _generatedContent[idx].text,
image: _generatedContent[idx].image,
isFromUser: _generatedContent[idx].fromUser ?? false,
);
},
itemCount: _generatedContent.length,
),
),
Padding(
padding: const EdgeInsets.symmetric(
vertical: 25,
horizontal: 15,
),
child: Row(
children: [
Expanded(
child: TextField(
autofocus: true,
focusNode: _textFieldFocus,
controller: _textController,
),
),
const SizedBox.square(
dimension: 15,
),
if (!_loading)
IconButton(
onPressed: () async {
await _testImagen(_textController.text);
},
icon: Icon(
Icons.image_search,
color: Theme.of(context).colorScheme.primary,
),
tooltip: 'Imagen raw data',
)
else
const CircularProgressIndicator(),
// NOTE: Keep this API private until future release.
// if (!_loading)
// IconButton(
// onPressed: () async {
// await _testImagenGCS(_textController.text);
// },
// icon: Icon(
// Icons.imagesearch_roller,
// color: Theme.of(context).colorScheme.primary,
// ),
// tooltip: 'Imagen GCS',
// )
// else
// const CircularProgressIndicator(),
],
),
),
],
),
),
);
}

Future<void> _testImagen(String prompt) async {
setState(() {
_loading = true;
});

var response = await _imagenModel.generateImages(prompt);

if (response.images.isNotEmpty) {
var imagenImage = response.images[0];
// Process the image
_generatedContent.add(
MessageData(
image: Image.memory(imagenImage.bytesBase64Encoded),
text: prompt,
fromUser: false,
),
);
} else {
// Handle the case where no images were generated
_showError('Error: No images were generated.');
}
setState(() {
_loading = false;
_scrollDown();
});
}
// NOTE: Keep this API private until future release.
// Future<void> _testImagenGCS(String prompt) async {
// setState(() {
// _loading = true;
// });
// var gcsUrl = 'gs://vertex-ai-example-ef5a2.appspot.com/imagen';

// var response = await _imagenModel.generateImagesGCS(prompt, gcsUrl);

// if (response.images.isNotEmpty) {
// var imagenImage = response.images[0];
// final returnImageUri = imagenImage.gcsUri;
// final reference = FirebaseStorage.instance.refFromURL(returnImageUri);
// final downloadUrl = await reference.getDownloadURL();
// // Process the image
// _generatedContent.add(
// MessageData(
// image: Image(image: NetworkImage(downloadUrl)),
// text: prompt,
// fromUser: false,
// ),
// );
// } else {
// // Handle the case where no images were generated
// _showError('Error: No images were generated.');
// }
// setState(() {
// _loading = false;
// });
// }

void _showError(String message) {
showDialog<void>(
context: context,
builder: (context) {
return AlertDialog(
title: const Text('Something went wrong'),
content: SingleChildScrollView(
child: SelectableText(message),
),
actions: [
TextButton(
onPressed: () {
Navigator.of(context).pop();
},
child: const Text('OK'),
),
],
);
},
);
}
}
Loading
Loading