diff --git a/examples/recipegenerator/README.md b/examples/recipegenerator/README.md new file mode 100644 index 00000000..9ccf18bb --- /dev/null +++ b/examples/recipegenerator/README.md @@ -0,0 +1,62 @@ +# Recipe Generator Application + +## Overview + +This Flask application leverages GPTScript and GPTScript Vision to suggest recipes based on an image uploaded by the user. By analyzing the image, the application can identify ingredients and propose a suitable recipe. + +## Features + +- Image upload functionality. +- Automatic ingredient recognition from images. +- Recipe suggestion based on identified ingredients. + +## Installation + +### Prerequisites + +- Python 3.8 or later +- Node.js and npm +- Flask +- Other Python and Node.js dependencies listed in `requirements.txt` and `package.json` respectively. + +### Steps + +1. Clone the repository: + + ``` bash + git clone https://github.com/gptscript-ai/gptscript.git + ``` + +2. Navigate to the `examples/recipegenerator` directory and install the dependencies: + + Python: + + ```bash + pip install -r requirements.txt + ``` + + Node: + + ```bash + npm install + ``` + +3. Setup `OPENAI_API_KEY` (Eg: `export OPENAI_API_KEY="yourapikey123456"`). You can get your [API key here](https://platform.openai.com/api-keys). + +4. Run the Flask application using `flask run` or `python app.py` + +## Usage + +1. Open your web browser and navigate to `http://127.0.0.1:5000/`. +2. Use the web interface to upload an image with some grocery items. +3. The application will process the image, identify potential ingredients, and suggest a recipe based on the analysis. +4. View the suggested recipe, try it and let us know how it turned out to be! + +## Under the hood + +Below are the processes that take place when you execute the application: + +- The Python app places the uploaded image as `grocery.png` in the current working directory. +- It then executes `recipegenerator.gpt` which internally calls `tools.gpt` to perform image analysis to identify the items from the uploaded image. +- The identified ingredients from the image will be stored in a `response.json` file. +- The recipegenerator will then read this response file, generate a recipe and add it to a recipe.md file. \ No newline at end of file diff --git a/examples/recipegenerator/app.py b/examples/recipegenerator/app.py new file mode 100644 index 00000000..db75c4aa --- /dev/null +++ b/examples/recipegenerator/app.py @@ -0,0 +1,44 @@ +from flask import Flask, request, render_template, jsonify +import subprocess +import os + +app = Flask(__name__) + +# Setting the base directory +base_dir = os.path.dirname(os.path.abspath(__file__)) +app.config['UPLOAD_FOLDER'] = base_dir + +SCRIPT_PATH = os.path.join(base_dir, 'recipegenerator.gpt') +GROCERY_PHOTO_FILE_NAME = 'grocery.png' # The expected file name +RECIPE_FILE_NAME = 'recipe.md' # The output file name + +@app.route('/', methods=['GET']) +def index(): + return render_template('index.html') + +@app.route('/upload', methods=['POST']) +def upload_file(): + if 'file' not in request.files: + return jsonify({'error': 'No file part'}), 400 + file = request.files['file'] + if file.filename == '': + return jsonify({'error': 'No selected file'}), 400 + if file: + filename = os.path.join(app.config['UPLOAD_FOLDER'], GROCERY_PHOTO_FILE_NAME) + file.save(filename) + try: + # Execute the script to generate the recipe + subprocess.Popen(f"gptscript {SCRIPT_PATH}", shell=True, stdout=subprocess.PIPE, cwd=base_dir).stdout.read() + + # Read recipe.md file + recipe_file_path = os.path.join(app.config['UPLOAD_FOLDER'], RECIPE_FILE_NAME) + with open(recipe_file_path, 'r') as recipe_file: + recipe_content = recipe_file.read() + + # Return recipe content + return jsonify({'recipe': recipe_content}) + except Exception as e: + return jsonify({'error': str(e)}), 500 + +if __name__ == '__main__': + app.run(debug=False) diff --git a/examples/recipegenerator/index.js b/examples/recipegenerator/index.js new file mode 100644 index 00000000..d25c475a --- /dev/null +++ b/examples/recipegenerator/index.js @@ -0,0 +1,104 @@ +import { Command, Option } from 'commander'; +import { fileTypeFromBuffer } from 'file-type'; +import { URL } from 'whatwg-url'; +import fs from 'fs'; +import OpenAI from 'openai'; + + +async function main() { + const program = new Command(); + + program.description('Utility for processing images with the OpenAI API'); + + program.addOption(new Option('--openai-api-key ', 'OpenAI API Key') + .env('OPENAI_API_KEY') + .makeOptionMandatory() + ); + + program.addOption(new Option('--openai-base-url ', 'OpenAI base URL') + .env('OPENAI_BASE_URL') + ); + + program.addOption(new Option('--openai-org-id ', 'OpenAI Org ID to use') + .env('OPENAI_ORG_ID') + ); + + program.addOption(new Option('--max-tokens ', 'Max tokens to use') + .default(2048) + .env('MAX_TOKENS') + ); + + program.addOption(new Option('--model ', 'Model to process images with') + .env('MODEL') + .choices(['gpt-4-vision-preview']) + .default('gpt-4-vision-preview') + ); + + program.addOption(new Option('--detail ', 'Fidelity to use when processing images') + .env('DETAIL') + .choices(['low', 'high', 'auto']) + .default('auto') + ); + + program.argument('', 'Prompt to send to the vision model'); + + program.argument('', 'List of image URIs to process. Supports file:// and https:// protocols. Images must be jpeg or png.'); + + program.action(run); + await program.parseAsync(); +} + +async function run(prompt, images, opts) { + let content = [] + for (let image of images) { + content.push({ + type: "image_url", + image_url: { + detail: opts.detail, + url: await resolveImageURL(image) + } + }) + } + + const openai = new OpenAI(opts.openaiApiKey, opts.baseUrl, opts.orgId); + const response = await openai.chat.completions.create({ + model: opts.model, + max_tokens: opts.maxTokens, + messages: [ + { + role: 'user', + content: [ + { type: "text", text: prompt }, + ...content + ] + }, + ] + }); + + console.log(JSON.stringify(response, null, 4)); +} + +async function resolveImageURL(image) { + const uri = new URL(image) + switch (uri.protocol) { + case 'http:': + case 'https:': + return image; + case 'file:': + const filePath = image.slice(7) + const data = fs.readFileSync(filePath) + const mime = (await fileTypeFromBuffer(data)).mime + if (mime != 'image/jpeg' && mime != 'image/png') { + throw new Error('Unsupported mimetype') + } + const base64 = data.toString('base64') + return `data:${mime};base64,${base64}` + default: + throw new Error('Unsupported protocol') + } +} + +main(); + + + diff --git a/examples/recipegenerator/package.json b/examples/recipegenerator/package.json new file mode 100644 index 00000000..f0adea04 --- /dev/null +++ b/examples/recipegenerator/package.json @@ -0,0 +1,20 @@ +{ + "name": "vision", + "version": "0.0.1", + "description": "Utility for processing images with the OpenAI API", + "exports": "./index.js", + "type": "module", + "scripts": { + "start": "node index.js" + }, + "bin": "index.js", + "keywords": [], + "author": "", + "dependencies": { + "commander": "^9.0.0", + "file-type": "^19.0.0", + "openai": "^4.28.0", + "whatwg-url": "^14.0.0" + } + } + \ No newline at end of file diff --git a/examples/recipegenerator/recipegenerator.gpt b/examples/recipegenerator/recipegenerator.gpt new file mode 100755 index 00000000..5d8d5823 --- /dev/null +++ b/examples/recipegenerator/recipegenerator.gpt @@ -0,0 +1,15 @@ +tools: sys.find, sys.read, sys.write, recipegenerator, tool.gpt + +Perform the following steps in order: + +1. Give me a list of 5 to 10 ingredients from the picture grocery.png located in the current directory and put those extracted ingredients into a list. +2. Based on these ingredients list, suggest me one recipe that is quick to cook and create a new recipe.md file with the recipe + + +--- +name: recipegenerator +description: Generate a recipe from the list of ingredients +args: ingredients: a list of available ingredients. +tools: sys.read + +Generate 1 new recipe based on the ingredients list \ No newline at end of file diff --git a/examples/recipegenerator/requirements.txt b/examples/recipegenerator/requirements.txt new file mode 100644 index 00000000..8882dd85 --- /dev/null +++ b/examples/recipegenerator/requirements.txt @@ -0,0 +1,2 @@ +Flask==2.0.1 +Werkzeug==2.2.2 diff --git a/examples/recipegenerator/templates/index.html b/examples/recipegenerator/templates/index.html new file mode 100644 index 00000000..62a80ebc --- /dev/null +++ b/examples/recipegenerator/templates/index.html @@ -0,0 +1,102 @@ + + + + + + Recipe Generator + + + + + + + + +
+

Recipe Generator

+
+

Don't know what to do with what's in your shopping cart? Well, click a picture and upload the image to Recipe Generator that will give you interesting ideas of what you can cook from those ingredients! This is built using GPTScript.

+
+
+ +
+
+
+
+ + +
+
+
+
+
+ +
+ + + + + diff --git a/examples/recipegenerator/tool.gpt b/examples/recipegenerator/tool.gpt new file mode 100644 index 00000000..6bb59bf8 --- /dev/null +++ b/examples/recipegenerator/tool.gpt @@ -0,0 +1,11 @@ +Name: vision +Description: Analyze a set of images using a given prompt and vision model. +Args: detail: Fidelity to process images at. One of ["high", "low", "auto"]. Default is "auto". +Args: max-tokens: The maximum number of tokens. Default is 2048. +Args: model: The name of the model to use. Default is "gpt-4-vision-preview". +Args: prompt: The text prompt based on which the GPT model will generate a response. +Args: images: Space-delimited list of image URIs to analyze. Valid URI protocols are "http" and "https" for remote images, and "file" for local images. Only supports jpeg and png. + +#!/bin/bash + +node index.js "${PROMPT}" ${IMAGES}