diff --git a/diffusers/unidiffuser.ipynb b/diffusers/unidiffuser.ipynb new file mode 100644 index 00000000..c01d85d9 --- /dev/null +++ b/diffusers/unidiffuser.ipynb @@ -0,0 +1,12164 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "PEQHAzAqkYp2" + }, + "source": [ + "# Generating images and text with UniDiffuser\n", + "\n", + "UniDiffuser was introduced in [One Transformer Fits All Distributions in Multi-Modal Diffusion at Scale](https://arxiv.org/abs/2303.06555).\n", + "\n", + "In this notebook, we will show how the [UniDiffuser pipeline](https://huggingface.co/docs/diffusers/api/pipelines/unidiffuser) in 🧨 diffusers can be used for:\n", + "\n", + "* Unconditional image generation\n", + "* Unconditional text generation\n", + "* Text-to-image generation\n", + "* Image-to-text generation\n", + "* Image variation\n", + "* Text variation\n", + "\n", + "One pipeline to rule six use cases 🀯\n", + "\n", + "Let's start!\n", + "\n", + "
\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pH5NByOqlJ39" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "lIYdn1woOS1n", + "outputId": "23733809-fa5b-4d10-8c56-80014c5be4ce" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Building wheel for diffusers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m104.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m219.1/219.1 kB\u001b[0m \u001b[31m27.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m105.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h" + ] + } + ], + "source": [ + "!pip install -q git+https://github.com/huggingface/diffusers\n", + "!pip install transformers accelerate -q" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P65FyJgBlRMQ" + }, + "source": [ + "## Unconditional image and text generation\n", + "\n", + "Throughout this notebook, we'll be using the [\"thu-ml/unidiffuser-v1\"](https://huggingface.co/thu-ml/unidiffuser-v1) checkpoint. UniDiffuser comes with two checkpoints:\n", + "\n", + "* [\"thu-ml/unidiffuser-v1\"](https://huggingface.co/thu-ml/unidiffuser-v1)\n", + "* [\"thu-ml/unidiffuser-v0\"](https://huggingface.co/thu-ml/unidiffuser-v0)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 876, + "referenced_widgets": [ + "2623bf67e4ac4aef8ea577c79e5cab5c", + "7b9b884d2e804d38acdf87b626cc3832", + "bee865ede2e641d4bab5a1aea9b2d1ab", + "ac94f1919b644a909eb1bf59b358a0a5", + "bf734b56c913411892f016456beb9b0c", + "50e19a3a830b4986b51cd2b8f874e73c", + "80f05bd27f00416dbd1d5a6164fe16c8", + "fbfc2dc1acd44525bebd64237a0c8aa8", + "d1f53d915f224a25a88305274cdc3d29", + "b0a8967d1249405b93f75d63ee8696e7", + "2a6454cfe82c4d1b8204955531ffecf8", + "1f10f9e73b69440392315c9ece5fee61", + "f98cd56f0af148b09eb76636abbb9e4e", + "e9f2d37964f847d5a738b335770b39f5", + "dcc236d9b6e14b5d8905f389f7f7c529", + "a8f820e0c61f49b9bfea784b836df22b", + "7d2722f497814317a648997784438b6a", + "d10a369680004e20ac841ce53527cb0c", + "17a73ddf0b43463190794be0ef16ef50", + "69d80b4a50a5434f9a2d5a74f6f6ed1c", + "cb57bdb83b5a4f298690de3e3ea45b53", + "45943a510dcd4c48a179bc55486a05bd", + "7fbd63399230475791bb5c860e31f334", + "99e5b93f77e4481c994b1e1936eae1f7", + "1daf1c9d46544c1e90f781e33a1465bf", + "86d35ba758a744ca8a5643c7572c5bca", + "0edfd5c64ad2428b9d6a633dd7fc514b", + "914065b19f504f9baa9ca0301bb6f219", + "db8e8548d11141878d112a23b6ca77d9", + "3e6fe31a09874f1aa719b0d828e0d154", + "5051a718bd9845ceb9b43c725014ef6b", + "4e49178fe4334834a934425452623834", + "14290d7888dc46708a48d2060d49f02e", + "5018d1d03df3403684afd87d5c80c41d", + "c19cf22e98434b3a9c54b862d25f862d", + "00f926ab2fee40baa0122a5cc3ad179b", + "6262fbac3cc9413790a6531228d75e34", + "b82b6e9c572146c087b87f978fb53787", + "8e3b173b001440688a011f95137dc1e3", + "b8a023889a3445aebfd8d759db811dc8", + "539d357f327b447bad05f3250cfd48e7", + "6f139c63e8714170b28e54151ec5e35d", + "ed79f430e1314137ba6a74e669f7b424", + "3afa3a9089ad4a8e92b36c955a5bf7bc", + "c2117c82b336423b9f95263314d26218", + "8cb05552eac740b7be3cb405096d4a70", + "bdc6a36f6d6a40789cd9098412f0c36a", + "6c94d44a34de4376a937a5ea91eff7e1", + "c45cfbe5de194774af5e5e525f405766", + "8504b35757594b9a968464076e75bf18", + "7edefe14c2474307b0d100ae7636d1bb", + "e4e95a57031343128370fa50d3547c71", + "50ef0503639b441890f96f0bbfe6430a", + "74547441217d4b78b561cd8c2aa44a22", + "a5d366a6deb3411394ad557cab49932d", + "41c1b0a602c54a7baa5b5f69f422f3b8", + "ce582edd3837468ca8e27b0ad05c1170", + "a30789decdec43b29cc63ebc4bc04421", + "03dc16c487ef418da3c17004ab3b7955", + "b46d128ebc384c8aaf471e0d3d9a0dad", + "f89a43dbbdc34dcc9ccf2a1a5b803c11", + "405f561e493e4801bbb9ab60fa06eb7b", + "780165d8c46c4211921e35e269c82e17", + "d635377a2cab435188ed87a4b0673f8f", + "275954fd560042a6b7fc7a6b9eed9811", + "c968bc1524124792a2058ce1ec8bb37a", + "e469c728ed9b4557b12ae1200954579a", + "d94c7c94e4934e2a84aa7a40cef9e3b6", + "52b48b9b8aaa4b8ba82dd5a661fe03b1", + "6e72967738ab4e539e49018e0a4cbf94", + "1e17fc537ec04e8bb0b19b1478a7d6f7", + "7ab7876fc7df46a59ed56a6129c73d73", + "eb42aeca75a24c79ad1c75e045d1c022", + "2bb1b0c565994ea0bbcbf5b0ad9aa2c4", + "8c56820303e94fc2be6247ceae7aa4fb", + "258269fe0b92478db6ec6a0ed7549c5d", + "bc8bbbd4a5964d83a30567a5a2d32848", + "f0f69e12e8554175b21d4858a65df43a", + "2ca4a64d08474060a858d6a904e86dd0", + "9109e6463281439882a10ee2c7bf30de", + "615a919b04f04457b2184957c70dd75f", + "97fbe31d2de440c08c367bea55488963", + "9ba8b9cecd894f2aa0ee601bb5a487d7", + "5d01d422ce5840f2829f7ba6b47241ad", + "16895d2fc770424fb4be69982746a19a", + "230f2e23d4494e23995248d5691ee751", + "9cd281357d764372bbed3120eecd4f6b", + "c10cd8a1a7364374ab89312b10cc0c24", + "aa121526f2f44ea2924790f63a070b8e", + "f6d604057664422490e0106e1fe63678", + "f1a5393819f44461af9a89b2970e58ef", + "f81dfe3b26a643cb99fb5ac4915788da", + "24e07354163346c49e456f740f9b927e", + "d71de1d221674b4f9ca77710853c528e", + "9d638ec5cbc14bc982d8ec4323b50504", + "360bb11270624e2190e3bdd6a8aee86c", + "18138b119b5e44958f32f79667180cd6", + "c7d4918e27174410a39df866fa41715c", + "8a04ca2f5c4a4ddf85331057f880f95f", + "4c89de4fcf07487e843b9519a0c59b3d", + "979ffac5812341d2b301ad2105df4663", + "c52caea913d64113bafc70e65da2deb5", + "ae4257fe27df4dddb28e78f1d189c4a0", + "914b04349d7f4f708485f860170ef36c", + "847e67cff7004b75a5ccf6613108b8d1", + "1f150fb8d2f0475e92300a50a1a112f5", + "e622b63e6f8e42568bd6c6fa7a8c2982", + "990e8194161a4efca091fb5a4be71929", + "96924c5af440416ba81f9052fbdd93c4", + "dad10a0c1405436c846c956a356b901e", + "297ccf531b3f4f31b1da61e9423ad71d", + "8a61bad1998b4190a5039845b35d0141", + "06a8ff54a81047a89f7431afd96e6f34", + "3dd848b725374d419db98b4bacf4d8a8", + "c28fd47e58f1413fae47bbecf229badc", + "07291caf41f14a77bfedd9a5432cce7d", + "68e63f2db37a47759c185e36c4fd8a39", + "87e6862a59cc4aa8ad944a0df7c6342a", + "a26cf397798446be87c1a28a502766f9", + "f4c4714d56074babadc19a63ece88bdb", + "84a5fbd4e74a40d7864996f8a9496b39", + "61c8de67b49849328dde6ab809090327", + "dda43252e3f54a7989468d926cb6759b", + "70f59f723c35499f88adc43c408320e1", + "b0791faae0eb4e7aa32a6329faf61d0f", + "0e2616f3ff6d479f92137ce595c2bcad", + "02c6f2cdef6d4bc5b639c97f846c3872", + "808f23e27103418abd1eeeb5a916154e", + "e72628d9e3ca46b88304bab6429d1b3c", + "4aab2eecde2b443dbbf1a22bf4a86cda", + "027abf26f7f44dd1bee8f6570d3a8d03", + "77a26d01cda1484cb83bb69a32383c4b", + "608ceb9800d74ca9afca31d8616ce43f", + "5fecc61f6734432ca33a4e29dc71d730", + "85cf1a15123b46ed9a5e777a8fe5f313", + "bb4531b14808401db4b8c6fc13ff8d07", + "4a313eab0771410d89d12f512d3612ae", + "6941865e4e3b405ea68d61da9ed593bd", + "2dad057884d246b1b4f22a154f5b6636", + "7c710594358b4d32907a1ccf85a6177d", + "ad4c548232534318b6ac927d22ed0c81", + "d8e3e068c4c74a028f9f3c92bc24f78c", + "cba3eb4871b04d3e9038173911af474a", + "34da22dcbc534e31bb0ade4ff7554cf4", + "61268d3026b7402ab75bd3c79a0708e0", + "51dabada73dd4f42964b045387a21f57", + "c70ef8d3faa94fb7852ca95c3e9c1f4e", + "f5158169573440828dd79ccf26bd6774", + "408325a4141c4b6886226ab44c90f8b0", + "2397c65aaf2b41b5a45d2c6c578ca987", + "7775ba190dbf4f9a93c93bb26be1104b", + "696548d775234b0999658ca3208b1a84", + "3ef235ec928048dca45212d724fc70c1", + "72729cf9b9744b639e751e84b9c5189c", + "5366bfcceaa44969b0e767e3955464eb", + "434068e40cb4499eaf68a71b97a6b521", + "f1272bf9207647feb94368c44eff597b", + "75478b8d502445b298a205a56d38ad61", + "23bbe04f615f4807859ffcdc580f9cf8", + "cd05513020d04a0b831fc5c1cfec20fe", + "b309e2dcb20447dd9c3e85c8abc46928", + "583a2aee6db14e6187dcde50f294e0fe", + "4d3e234713b64f93a741dd730c03455c", + "8f907c30ae724ba5a5109ab699499454", + "db053cfc7a5d4368be356640f04b2277", + "b0755018085b4c3b9a5cd0ef78304d15", + "514aa5419cf94790b30b697f998c25b7", + "83afe6cbce6f487c9f67df23870e02a0", + "7090e5c74cee4a00b5337fae185f20e9", + "61cdb5a02f8d4e7ab94ef8d1c615b3ed", + "70ecc0aa7008430ea9c4624ff2737d2b", + "a29f87515786457b9e3ee8df2d4375a7", + "666a86126f364ad6a7ea56e1ca6fd5ab", + "d289d7f706e64f18b8e9627d2392145d", + "40a4a961c98e4797bd4d810481637457", + "2c39050294b541f3bfc53379b7bd5777", + "3f1ee951d03844578f18173a31ee3703", + "824c49057eab4ea2acfa77c30c923f30", + "4dab2941fef2410d847cb87a019510b2", + "74ab1c7e9150464e99f869e5fc1707e6", + "9bf522ed0d914cd29df3ef2f2f151b41", + "855e73754b7642268b5e5d2040143486", + "0aa84589457a4c0aaa66e1a566c331a3", + "4f5e1a1ddc3d4c5a98c95d720787115f", + "542340d6aa244aedb8b0f42a754dca9d", + "6ded642ca0874be7a8e0a8d5e0f2a8fe", + "72e8dc8ea246415cb01c4e6d0a9265a3", + "3d061a5eb8f443fba871973841012129", + "04a28462afad467ea3653d767de78661", + "bfc60a39cc03405eb19a35e259ad05e6", + "f21cc95bb9164e73a5581c3d80fc085b", + "cec0b861849043c0a3931334dc60c60d", + "379487eb35a844a6807dbd65954656a2", + "3c9fbae86ea043edae3184aa81ab70f1", + "ca3c0f64f4094e65b4c6ff96a07b6843", + "bac07abe4747451ab45b5ebdabd4af57", + "8d518442d2a2434d95fa49d439072bb6", + "730fc9d5cf294ed6897c0920f98e1452", + "1bbaed02c2ef44e692f524c3a1b0c83f", + "4d44e17bfa674daba6e70990be96a21f", + "c59041e2a8b341f3bda76bb54892fd08", + "8bfb55bb11e74f4e8f30f2015fa7245d", + "1af530e2fad24111940a387737b3b0e7", + "33a77176648e4603ac1b22b59ff20fd7", + "64c4908ee3ca415dbca906b2cfcd6241", + "0e0d30da42f54f99b1902a9dc982e37e", + "f310ddc229c14e008f89dbdf7842b020", + "1a9fd29cea2f47629679a9d75982de77", + "2b91b5647c6a44e28c759ded08b5503b", + "90279c703b2644afb9f3b253e2ec6ed1", + "700b7968670546a399813a8a3fbdf39c", + "8e25172e953a48a79f4dc21782539bb5", + "2b85e2a9e40a4c7894d388f744d28f0d", + "5083496a553a4d899e00e302eaf69081", + "0f9d985ee8344bc780a27940a4cb6f76", + "8a00478658a5482d99a0d37f9fd70e19", + "32680c5f367f4cd0bcea3702540a7e6c", + "1f9e4d5244504ddcab8ebf1527993fa2", + "3e821d5e8f084906a2aaaa57bd5551a3", + "5d2ce906f23a4147884309d347cbc660", + "a324f4a4b6c64558a1ebc7a7480db9e3", + "c0d98f6e5c3146bba1ab5149fc7b30d8", + "7067648b6f0a474a9cac186ef3ce4bb7", + "d0f895ab222947249d533cfc6a5e78ac", + "14e2994d557d4952a09817beaa5ed79d", + "5df2761e8c2545dbaea48af73e60d5ec", + "7ddb91d1e97e4acab2cf7cf74b2fd3e2", + "894a2fe2daa64e6b93265cb6d9ca7a97", + "11a9a7553b8e446eb5e58246b406a313", + "f8741d34cdb245159c15d9f749b57ffc", + "d724c9bb931847afadaed628b3420a98", + "9047410dc4a54e51be22d2d4735db2ec", + "adb350df3b9840459ba5d216848e702f", + "f277ec55253941029b3092bbf930d9cb", + "fef592afa88a4e6c9b151e0b0c7125ff", + "98946d10a3824701a102ca505c56d3a6", + "f181f295ce044fbf9f3cba4027ba8356", + "6136abfa39a14afba03b588661a3ec03", + "c3b94e7958c84a9f8f38391a8da7802a", + "5416843f5fa44f7c82b6d38ef373661d", + "f33404aac16e41038b80db4985f4b106", + "26e664b7827f445cb78333108c172560", + "2b6b544a0ac04ddea7cfb401357326b1", + "8ebf07e263ba48dd939695a5ec35cc40", + "6c4dcf28aead4273a7fb53836e5fe958", + "bcc343c10c684b96b3d07eba3c2da1fd", + "6911020dc3ff49399e979cb31a484be2", + "48f21855fd5545a3a4a414ad1111e07c", + "79e78515859a4b618645a8fdf7166f02", + "aa4386cb1297430f9da17be5ff63ea6c", + "549dd628b1a64f04b1607855b4f55152", + "d5ab39677e244ccc941cd31e52ec7d8a", + "ceca35d1dd7b4244ba547c5047eb6351", + "690fbd8adadc476b9a8dc5bc2c536569", + "4e30e03e1204419693bc36ab6046c992", + "3d98eb1ce26c443aa93c118a84c0f775", + "5f85f9feceff4c288de0a700f1f31edd", + "79e75067e4fc471dafc20820dece8c3e", + "adb9561eccd44096995d8a8c9c8d6ccd", + "2af301ffcfab4bc89bbe10658c722768", + "06aa64c337904c4b8a5c64a4f22770a6", + "5bf1e08ffc5447edbae9433b12276da4", + "546836834e7843bfa9e33c10cbe3a6f9", + "111c471b394b4ecc97cfb2804743075a" + ] + }, + "id": "oc4Wtedcem_z", + "outputId": "8b8f260f-a236-42b0-df90-342db791225b" + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2623bf67e4ac4aef8ea577c79e5cab5c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading (…)ain/model_index.json: 0%| | 0.00/694 [00:00