From 4bcdb597bc7a6dafa1ae3e143315c03168e12489 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 4 Jan 2023 04:13:32 -0800 Subject: [PATCH] [Impeller Scene] Compute joint transforms and apply them to skinned meshes --- ci/licenses_golden/licenses_flutter | 4 + impeller/fixtures/two_triangles.glb | Bin 10180 -> 10048 bytes .../renderer/backend/gles/texture_gles.cc | 28 ++++ impeller/renderer/backend/metal/formats_mtl.h | 8 ++ impeller/renderer/backend/vulkan/formats_vk.h | 10 ++ impeller/renderer/formats.h | 6 + impeller/scene/BUILD.gn | 2 + impeller/scene/animation/animation_player.cc | 2 +- impeller/scene/animation/property_resolver.cc | 12 +- impeller/scene/geometry.cc | 25 ++++ impeller/scene/geometry.h | 6 + impeller/scene/importer/importer_gltf.cc | 52 +++++--- impeller/scene/importer/importer_unittests.cc | 2 + impeller/scene/importer/scene.fbs | 1 + impeller/scene/importer/vertices_builder.cc | 15 +-- impeller/scene/mesh.cc | 7 +- impeller/scene/mesh.h | 6 +- impeller/scene/node.cc | 45 +++++-- impeller/scene/node.h | 13 +- impeller/scene/scene.cc | 4 +- impeller/scene/scene_context.cc | 7 +- impeller/scene/scene_unittests.cc | 25 ++++ impeller/scene/shaders/skinned.vert | 53 ++++++-- impeller/scene/skin.cc | 121 ++++++++++++++++++ impeller/scene/skin.h | 42 ++++++ 25 files changed, 429 insertions(+), 67 deletions(-) create mode 100644 impeller/scene/skin.cc create mode 100644 impeller/scene/skin.h diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 34c5ab34c30d1..02e8ce98e055c 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1647,6 +1647,8 @@ ORIGIN: ../../../flutter/impeller/scene/scene_encoder.h + ../../../flutter/LICEN ORIGIN: ../../../flutter/impeller/scene/shaders/skinned.vert + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/shaders/unlit.frag + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/shaders/unskinned.vert + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/scene/skin.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/scene/skin.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart + ../../../flutter/LICENSE @@ -4116,6 +4118,8 @@ FILE: ../../../flutter/impeller/scene/scene_encoder.h FILE: ../../../flutter/impeller/scene/shaders/skinned.vert FILE: ../../../flutter/impeller/scene/shaders/unlit.frag FILE: ../../../flutter/impeller/scene/shaders/unskinned.vert +FILE: ../../../flutter/impeller/scene/skin.cc +FILE: ../../../flutter/impeller/scene/skin.h FILE: ../../../flutter/impeller/tessellator/c/tessellator.cc FILE: ../../../flutter/impeller/tessellator/c/tessellator.h FILE: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart diff --git a/impeller/fixtures/two_triangles.glb b/impeller/fixtures/two_triangles.glb index 4c7b8803bc92781621e8ce3607149971becdaba0..0ecae588ca88c89a58173ea80aebe42319d4c20c 100644 GIT binary patch literal 10048 zcmeHMTWlOx8J4;}1*K=Ktm!M7*)w%?uIGGeFg=*g^9laEW>!mvj?U7cz;7n_Y7qnQ zLqZ)@`Jtr^URNt<;}@KF30^l=GZX#y%nykPeqFWTMj$ksq`OZ;pVqYfLeDmTwwE}065jTo%NK5*94GTiVP$k{c%ypwtRQ&YR z?8N9i2u96vJ7s91u^8Fdj9hF+J}yIYsxi(yByrOz^n+@o1@^zG#u?3`Z5KOc^OBUX z@;a8Jl=3CNJ4QM-LypbxFO=@Q@-LL`7&(+A);2av4cl#fhg>mg%Vlk$w4$x>LzY?7 z_P76ev|t-rHW7EVTrL&ojoRv(u2r!QZ1Br!RU0?ThIv9QI)~zt=o|#JOIPZI#w+Y* zW`n!mHYhv0U~BzJQ^ScA#{(fjKE(3C@wU3=oVxqx?09XNYHZysRZ3RreEp?rS!QXu zX3^sYfB)RVRAFjv);U@V6Q3HNo0~s=MsSXm*}3_d(P@4N^;j64ot&60@I%<-PtHxv z78a-vw(d_(Oig~=eL}UMZdXc08ad-7sD22x(#>LYpKdO}H+1tpT@k_eghd=^K4nqz zON;QaQ5=|uTP<`NH#UCyrqRaW>Wqp|IPQv=$~gn8j`|?_h-dvs51!FQ^sXAFJJNFP z>{-n`RnorTM6YO6)(u_Lt-?d=IG1HnLJ*2ZO~*0gxb#J5Y2QiN=Mc7DZ*X*b^X3{k~9nnUZqikn7jWBth zN7*>5qwGMC+p(RF;S@R33285kk51$K4@Vt^+Cjadx+zYDQW{D*zHU)po~zLyV-a+a z^4QS?W8HMkj%w3N>zW?T5iQLqz3UWHq}(p)Qio~}@|y&% zL%9d}O(NH!W;^mb6m5{-#QzRe8|1@(q{};q{a*pj7r^pEmNu>FtMqF;g{jf=+}X3v zUtTB^^AVNs5u z9E!6fgoK?RN^a63kq$>oqG@#NbvR8DO`}_KL^&HnIUKK)5JNefsZ^X`N0c)$ltY=q z`4@~=yG)5vs5sM-^Lw^MVnZuPmfNkja71L;`5D_{X!6=$>2k*T%T24^p>-kW+PNL^ zBp=6SnMt*CC7la(&$jhELmR7qbWzI28eYtcvHDA?c>YMac>bhh z__KdF)DH&#X^@Bh>CJE6#c^X(v-s=$7ITf;!u@3rbKHM!Jc4Ixr;GYEyZhUk;RY`- zzSGgC=X)mDxc0gAhW5{gIQriPx}1I^1Yq2a8;^|Ow&m(^b&QsGX-?$vYJj2~#D?pt+#aBqz7wq4#G7{-_*9JlAx z!QXF@8(JJN+GwnXdD`wfjD{=RkH);byNhu-1z1^JTvRXy?oZ-*7kCf&4d6F{Id*yX z4z{>>4|IBf2Y^X;1o$NIBJk&c&jHsQUltc1#q$pE<7j&l_#41a13wGA3w#Cmd%!;i z-UI#_@T6vBOlUiO}k?AS1c@O_th*$Y>;ZJ2T|@GVFB*uuXawP6b3 z&)(k8lG|UhVagfc@5~Rd$xO|LDc=Pa1`=%S${8D`90fl9@*sQt`Y{`(OaMPKAhBzA za5hYN7WnsXr`bmz`^ryX#|VFTIK?hqdUY6f^+Uk7zI~Lv`R9HGX5R)ZPbS%gjRzE% zodKSlIKu8acTR!XCxD-Ot)D&fmxmRY{XFm=_xjjv+yw<@{|We&Tl?7ecP}b1`x>zN zwO;ny>)Q&+w=3W`qx+Z92K5z-+>=J>JV+{KvKevk43D?PJfq zcu|4bgil@VW1qSEf&#M%>p$;jzshbXFq^QPJ;J8n_n-o^)4+Epl1%&EyaKZcKmF6A zZ02}Uf!UXUNB$tOd!PEvFzhPfV~1sS=J3Bi278I{w0ukD5PsrXAG3b>WgDgt{^6Z{ z?9Q)Wv|$S2e?HO6-v8Ha8>SH6L`?oRuw%m%!r^-J@0IwC!^o(u8|>%4o7nGLH@b2A V`^S7IT>m(&a5v+3)2G$={{>7S@NxhE literal 10180 zcmeHMYit}>6d< zW@p#CI~4d4d6KLB?w#|UbM86kK6X7B9Xj#>m&^6*2AAt&pLV&94fglVF=8SisZ84( zGbzO-MO0;lX=A=TrO0tPQ9B7r?ct~tAD5Kc?z{SHXS^-G7GH?*Ff)>ph{$mW^DQhh z?_m;QC}7&yG)gdSBXdk#jA4u!G~RibxI8Y=7v9nN(OfSpHY86&vm&daYN#GzVas@Y ztT*KM`u)6zZSk|l;|~Q`U%=-JdYUt192er*5FZQ$AZBO6U?MC=;m!z$*QVZ3PB8}o zRT1NfXx6%?nQO_Y^!wS?AjkWqX13MCj?R1LnDA62I<82F!3ayIW6t%ktg3P>e;)AA z?oQ5YHaC<1(nP3pF)1-+WFzg?kP;E&lTm~paX1-))2wX{yoZJ$E#9ugL^Y{M3{8|6 zkD!N?m`M#!iSf7;H9HA0HXW5vG)TNEDwAe0(L^(CoQIhfRk)EC*m-IzRvam5agt43I%919Wh!RGlGs8A*T%9s75FA;JNwqD%`Ecw%n$fU-%BIu~$;* z#*aO_cuDeDd1XtI_ga!)8N=t8;djiiE|ki=vM!X$7(tXI);2av3EORYha8uaV^L`^ zGA@lXZK{%#vYYGxMz9TyDTuomjYh%)a&mGiE+w!J%rRqPLh6vCvT{TW8^}}&>`C8|9HTWhtwdrGH8PWa zDXOXx8B40PJICDJKiECg-QQ;%Ekm8hJNo+vzVtQLI9B@l2YTCkm^ReoPFYC+wON5WKdAeW%@A-I#>R%o}_4SJZ2-H^pDhv4T7 zb2!i}mO0*PnuCv-;=nx2YN4lbX5*)~D(Ns>?G+IU!(9$jxlhKbqcVs-;#u^J4nsO2 zE2gHgRy?9FHJ`gt&0nQ)fe7IW~=2b%pb5Cc77Z*Pis>K*)U&dID%@?qd&WhQEwlR~6*}ekX2y-FZ zm`)~4InN8(*k}vcc|k74HU#Y{GNxm*J=oFSgY(}WbrfnD>oL(xvDcPTTgq|2Mt$X6 zEetXiK^apyc430CZYpLM+NLY5VtQeY=+Z2tSDa$bNp%*nUbH+NB$;jmh3@BS4DO{v z-`hS^#D19)MI6i`VOv-q7sz{jC<=31*x_Z`@DAn46|L`_=P;csnj&d0Q{H*>A`>rD zvU&6(*2~mv89HC4X7lKU&hvO*D(QLjLg(qVr4-!<@6uaCOJ2qUf!6nzbG~S~=WD~2?x31_ zZLM^F!1(26Cu!Td@CQq|UEoQ`i6;Tpi6?=Q0p#e$VV!2cTgnw*&?eqm((?e_SgqPQ zzSWU>F2p&K=e^GS;e*cn@%rIU_Hala4A#>i1pA}&wYA^7-Q_ym-G{#rq=?yEDSYZV z>~j5U`xN?goIX^p*r#=@7|!tW^c#W}dzL2^7)b8U5%>J@;6;o)Jb2cM6kzY2UA z_!{t=z%KakQ8GOIN!X~R{s8-d4*?$o9tJ)Ed>?Sq@MC!R8|W_pe;Z?u0-pnZ9QcR8 ztH4hKKL@-4d>MEX_$A=0z}N7(``pSAu-EWzKYvZ{#^Zk*59^+}i_Np+R*Hl59Nfj` zec)D#gL<%m{QmTUfTmG=(cM5EySX5sX%s(r!%hAwEefwmiyFmmb-Brv0}lzC2OiQW zesQ3l+&1%w@S&MUG>Tt-@lKLzJ|~=NKBv6|eCN^6k&*Ey1T?JpZv`~%4&Xn#K1hySzbSNGzo}8Ya^br0_49iPn!XJD_zTYq{WI01Z>C!R z0WkmWs_>ho8uIc|jm`o8Zg@qwc$y(UKF#P9ACH|Co{QFz^=O^`72u8A&Iqw5>Is_u zGhlXVNtlYc$z;r}-v#^*vnV|LhMS=2Zv+47%>_YgY#^5!8+3}VB384AQ}SSgPH_*$ zUd4Mfo#KYtMWJ!aO+LTn*4@B&pI8z;CA-N1*{xH&@z%qF^i(|=f2v-m`2Mc5!pUSE zLDMO|{*e{oSIZ3fgTTiZOdw?6K8S?t(itxA16^-K8zFJ3aK66%h=b5wGUw~izZasO-b4K`%=Ztm( z_=^!Y`RUw}@a)`@M)5yicas;I7KJNKi`qYcM-iK+|GprgX%yS*&AM08yZ3w9`*(S4 q<=QdrXIkb?Z-^)oYXtDc>3CHmiw|NjQXJ+ ToRenderBufferFormat(PixelFormat format) { case PixelFormat::kB8G8R8A8UNormInt: case PixelFormat::kR8G8B8A8UNormInt: return GL_RGBA4; + case PixelFormat::kR32G32B32A32Float: + return GL_RGBA32F; + case PixelFormat::kR16G16B16A16Float: + return GL_RGBA16F; case PixelFormat::kS8UInt: return GL_STENCIL_INDEX8; case PixelFormat::kD32FloatS8UInt: diff --git a/impeller/renderer/backend/metal/formats_mtl.h b/impeller/renderer/backend/metal/formats_mtl.h index fc4956bec212d..056c9a73146ac 100644 --- a/impeller/renderer/backend/metal/formats_mtl.h +++ b/impeller/renderer/backend/metal/formats_mtl.h @@ -29,6 +29,10 @@ constexpr PixelFormat FromMTLPixelFormat(MTLPixelFormat format) { return PixelFormat::kR8G8B8A8UNormInt; case MTLPixelFormatRGBA8Unorm_sRGB: return PixelFormat::kR8G8B8A8UNormIntSRGB; + case MTLPixelFormatRGBA32Float: + return PixelFormat::kR32G32B32A32Float; + case MTLPixelFormatRGBA16Float: + return PixelFormat::kR16G16B16A16Float; case MTLPixelFormatStencil8: return PixelFormat::kS8UInt; case MTLPixelFormatDepth32Float_Stencil8: @@ -57,6 +61,10 @@ constexpr MTLPixelFormat ToMTLPixelFormat(PixelFormat format) { return MTLPixelFormatRGBA8Unorm; case PixelFormat::kR8G8B8A8UNormIntSRGB: return MTLPixelFormatRGBA8Unorm_sRGB; + case PixelFormat::kR32G32B32A32Float: + return MTLPixelFormatRGBA32Float; + case PixelFormat::kR16G16B16A16Float: + return MTLPixelFormatRGBA16Float; case PixelFormat::kS8UInt: return MTLPixelFormatStencil8; case PixelFormat::kD32FloatS8UInt: diff --git a/impeller/renderer/backend/vulkan/formats_vk.h b/impeller/renderer/backend/vulkan/formats_vk.h index f0d56741b3255..10441c3624e62 100644 --- a/impeller/renderer/backend/vulkan/formats_vk.h +++ b/impeller/renderer/backend/vulkan/formats_vk.h @@ -148,6 +148,10 @@ constexpr vk::Format ToVKImageFormat(PixelFormat format) { return vk::Format::eB8G8R8A8Unorm; case PixelFormat::kB8G8R8A8UNormIntSRGB: return vk::Format::eB8G8R8A8Srgb; + case PixelFormat::kR32G32B32A32Float: + return vk::Format::eR32G32B32A32Sfloat; + case PixelFormat::kR16G16B16A16Float: + return vk::Format::eR16G16B16A16Sfloat; case PixelFormat::kS8UInt: return vk::Format::eS8Uint; case PixelFormat::kD32FloatS8UInt: @@ -178,6 +182,12 @@ constexpr PixelFormat ToPixelFormat(vk::Format format) { case vk::Format::eB8G8R8A8Srgb: return PixelFormat::kB8G8R8A8UNormIntSRGB; + case vk::Format::eR32G32B32A32Sfloat: + return PixelFormat::kR32G32B32A32Float; + + case vk::Format::eR16G16B16A16Sfloat: + return PixelFormat::kR16G16B16A16Float; + case vk::Format::eS8Uint: return PixelFormat::kS8UInt; diff --git a/impeller/renderer/formats.h b/impeller/renderer/formats.h index a35378dbde1ec..12015586c992f 100644 --- a/impeller/renderer/formats.h +++ b/impeller/renderer/formats.h @@ -87,6 +87,8 @@ enum class PixelFormat { kR8G8B8A8UNormIntSRGB, kB8G8R8A8UNormInt, kB8G8R8A8UNormIntSRGB, + kR32G32B32A32Float, + kR16G16B16A16Float, // Depth and stencil formats. kS8UInt, @@ -290,6 +292,10 @@ constexpr size_t BytesPerPixelForPixelFormat(PixelFormat format) { return 4u; case PixelFormat::kD32FloatS8UInt: return 5u; + case PixelFormat::kR16G16B16A16Float: + return 8u; + case PixelFormat::kR32G32B32A32Float: + return 16u; } return 0u; } diff --git a/impeller/scene/BUILD.gn b/impeller/scene/BUILD.gn index 69c024f9e4232..dbd531cf8e17d 100644 --- a/impeller/scene/BUILD.gn +++ b/impeller/scene/BUILD.gn @@ -31,6 +31,8 @@ impeller_component("scene") { "scene_context.h", "scene_encoder.cc", "scene_encoder.h", + "skin.cc", + "skin.h", ] public_deps = [ diff --git a/impeller/scene/animation/animation_player.cc b/impeller/scene/animation/animation_player.cc index d2e9c353e03e1..01119fb521f61 100644 --- a/impeller/scene/animation/animation_player.cc +++ b/impeller/scene/animation/animation_player.cc @@ -54,7 +54,7 @@ void AnimationPlayer::Update() { void AnimationPlayer::Reset() { for (auto& [node, transform] : default_target_transforms_) { - node->SetLocalTransform(transform); + node->SetLocalTransform(Matrix()); } } diff --git a/impeller/scene/animation/property_resolver.cc b/impeller/scene/animation/property_resolver.cc index d4f5e3c7da830..d9d5db94e44e6 100644 --- a/impeller/scene/animation/property_resolver.cc +++ b/impeller/scene/animation/property_resolver.cc @@ -89,8 +89,8 @@ void TranslationTimelineResolver::Apply(Node& target, if (key.lerp < 1) { value = values_[key.index - 1].Lerp(value, key.lerp); } - target.SetLocalTransform(Matrix::MakeTranslation(value * weight) * - target.GetLocalTransform()); + target.SetLocalTransform(target.GetLocalTransform() * + Matrix::MakeTranslation(value * weight)); } RotationTimelineResolver::RotationTimelineResolver() = default; @@ -108,8 +108,8 @@ void RotationTimelineResolver::Apply(Node& target, if (key.lerp < 1) { value = values_[key.index - 1].Slerp(value, key.lerp); } - target.SetLocalTransform(Matrix::MakeRotation(value * weight) * - target.GetLocalTransform()); + target.SetLocalTransform(target.GetLocalTransform() * + Matrix::MakeRotation(value * weight)); } ScaleTimelineResolver::ScaleTimelineResolver() = default; @@ -125,8 +125,8 @@ void ScaleTimelineResolver::Apply(Node& target, SecondsF time, Scalar weight) { if (key.lerp < 1) { value = values_[key.index - 1].Lerp(value, key.lerp); } - target.SetLocalTransform(Matrix::MakeScale(value * weight) * - target.GetLocalTransform()); + target.SetLocalTransform(target.GetLocalTransform() * + Matrix::MakeScale(value * weight)); } } // namespace scene diff --git a/impeller/scene/geometry.cc b/impeller/scene/geometry.cc index f39bd7b3b6355..099476d14202d 100644 --- a/impeller/scene/geometry.cc +++ b/impeller/scene/geometry.cc @@ -12,6 +12,8 @@ #include "impeller/geometry/vector.h" #include "impeller/renderer/device_buffer_descriptor.h" #include "impeller/renderer/formats.h" +#include "impeller/renderer/sampler_descriptor.h" +#include "impeller/renderer/sampler_library.h" #include "impeller/renderer/vertex_buffer.h" #include "impeller/renderer/vertex_buffer_builder.h" #include "impeller/scene/importer/scene_flatbuffers.h" @@ -117,6 +119,8 @@ std::shared_ptr Geometry::MakeFromFlatbuffer( return MakeVertexBuffer(std::move(vertex_buffer), is_skinned); } +void Geometry::SetJointsTexture(const std::shared_ptr& texture) {} + //------------------------------------------------------------------------------ /// CuboidGeometry /// @@ -239,10 +243,31 @@ void SkinnedVertexBufferGeometry::BindToCommand( command.BindVertices( GetVertexBuffer(*scene_context.GetContext()->GetResourceAllocator())); + SamplerDescriptor sampler_desc; + sampler_desc.min_filter = MinMagFilter::kNearest; + sampler_desc.mag_filter = MinMagFilter::kNearest; + sampler_desc.mip_filter = MipFilter::kNone; + sampler_desc.width_address_mode = SamplerAddressMode::kRepeat; + sampler_desc.label = "NN Repeat"; + + SkinnedVertexShader::BindJointsTexture( + command, + joints_texture_ ? joints_texture_ : scene_context.GetPlaceholderTexture(), + scene_context.GetContext()->GetSamplerLibrary()->GetSampler( + sampler_desc)); + SkinnedVertexShader::VertInfo info; info.mvp = transform; + info.enable_skinning = joints_texture_ ? 1 : 0; + info.joint_texture_size = + joints_texture_ ? joints_texture_->GetSize().width : 1; SkinnedVertexShader::BindVertInfo(command, buffer.EmplaceUniform(info)); } +// |Geometry| +void SkinnedVertexBufferGeometry::SetJointsTexture( + const std::shared_ptr& texture) { + joints_texture_ = texture; +} } // namespace scene } // namespace impeller diff --git a/impeller/scene/geometry.h b/impeller/scene/geometry.h index b38850080b6e6..269fe46376ad4 100644 --- a/impeller/scene/geometry.h +++ b/impeller/scene/geometry.h @@ -45,6 +45,8 @@ class Geometry { HostBuffer& buffer, const Matrix& transform, Command& command) const = 0; + + virtual void SetJointsTexture(const std::shared_ptr& texture); }; class CuboidGeometry final : public Geometry { @@ -119,8 +121,12 @@ class SkinnedVertexBufferGeometry final : public Geometry { const Matrix& transform, Command& command) const override; + // |Geometry| + void SetJointsTexture(const std::shared_ptr& texture) override; + private: VertexBuffer vertex_buffer_; + std::shared_ptr joints_texture_; FML_DISALLOW_COPY_AND_ASSIGN(SkinnedVertexBufferGeometry); }; diff --git a/impeller/scene/importer/importer_gltf.cc b/impeller/scene/importer/importer_gltf.cc index 2b6ab6322c69c..a6f10b180f298 100644 --- a/impeller/scene/importer/importer_gltf.cc +++ b/impeller/scene/importer/importer_gltf.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -199,22 +200,22 @@ static void ProcessNode(const tinygltf::Model& gltf, /// Matrix transform; - if (in_node.translation.size() == 3) { - transform = transform * Matrix::MakeTranslation( - {static_cast(in_node.translation[0]), - static_cast(in_node.translation[1]), - static_cast(in_node.translation[2])}); + if (in_node.scale.size() == 3) { + transform = + transform * Matrix::MakeScale({static_cast(in_node.scale[0]), + static_cast(in_node.scale[1]), + static_cast(in_node.scale[2])}); } if (in_node.rotation.size() == 4) { transform = transform * Matrix::MakeRotation(Quaternion( in_node.rotation[0], in_node.rotation[1], in_node.rotation[2], in_node.rotation[3])); } - if (in_node.scale.size() == 3) { - transform = - transform * Matrix::MakeScale({static_cast(in_node.scale[0]), - static_cast(in_node.scale[1]), - static_cast(in_node.scale[2])}); + if (in_node.translation.size() == 3) { + transform = transform * Matrix::MakeTranslation( + {static_cast(in_node.translation[0]), + static_cast(in_node.translation[1]), + static_cast(in_node.translation[2])}); } if (in_node.matrix.size() == 16) { if (!transform.IsIdentity()) { @@ -317,11 +318,14 @@ static void ProcessAnimation(const tinygltf::Model& gltf, fb::AnimationT& out_animation) { out_animation.name = in_animation.name; - std::vector> channels; + // std::vector channels; + std::vector translation_channels; + std::vector rotation_channels; + std::vector scale_channels; for (auto& in_channel : in_animation.channels) { - auto out_channel = std::make_unique(); + auto out_channel = fb::ChannelT(); - out_channel->node = in_channel.target_node; + out_channel.node = in_channel.target_node; auto& sampler = in_animation.samplers[in_channel.sampler]; /// Keyframe times. @@ -349,7 +353,7 @@ static void ProcessAnimation(const tinygltf::Model& gltf, const float* time_p = reinterpret_cast( times_buffer.data.data() + times_bufferview.byteOffset + times_accessor.ByteStride(times_bufferview) * time_i); - out_channel->timeline.push_back(*time_p); + out_channel.timeline.push_back(*time_p); } } @@ -387,7 +391,8 @@ static void ProcessAnimation(const tinygltf::Model& gltf, keyframes.values.push_back( fb::Vec3(value_p[0], value_p[1], value_p[2])); } - out_channel->keyframes.Set(std::move(keyframes)); + out_channel.keyframes.Set(std::move(keyframes)); + translation_channels.push_back(std::move(out_channel)); } else if (in_channel.target_path == "rotation") { if (values_accessor.type != TINYGLTF_TYPE_VEC4) { std::cerr << "Unexpected type \"" << values_accessor.type @@ -404,7 +409,8 @@ static void ProcessAnimation(const tinygltf::Model& gltf, keyframes.values.push_back( fb::Vec4(value_p[0], value_p[1], value_p[2], value_p[3])); } - out_channel->keyframes.Set(std::move(keyframes)); + out_channel.keyframes.Set(std::move(keyframes)); + rotation_channels.push_back(std::move(out_channel)); } else if (in_channel.target_path == "scale") { if (values_accessor.type != TINYGLTF_TYPE_VEC3) { std::cerr << "Unexpected type \"" << values_accessor.type @@ -421,15 +427,22 @@ static void ProcessAnimation(const tinygltf::Model& gltf, keyframes.values.push_back( fb::Vec3(value_p[0], value_p[1], value_p[2])); } - out_channel->keyframes.Set(std::move(keyframes)); + out_channel.keyframes.Set(std::move(keyframes)); + scale_channels.push_back(std::move(out_channel)); } else { std::cerr << "Unsupported animation channel target path \"" << in_channel.target_path << "\". Skipping." << std::endl; continue; } } + } - channels.push_back(std::move(out_channel)); + std::vector> channels; + for (const auto& channel_list : + {translation_channels, rotation_channels, scale_channels}) { + for (const auto& channel : channel_list) { + channels.push_back(std::make_unique(channel)); + } } out_animation.channels = std::move(channels); } @@ -458,6 +471,9 @@ bool ParseGLTF(const fml::Mapping& source_mapping, fb::SceneT& out_scene) { const tinygltf::Scene& scene = gltf.scenes[gltf.defaultScene]; out_scene.children = scene.nodes; + out_scene.transform = + ToFBMatrixUniquePtr(Matrix::MakeScale(Vector3(1, 1, -1))); + for (size_t texture_i = 0; texture_i < gltf.textures.size(); texture_i++) { auto texture = std::make_unique(); ProcessTexture(gltf, gltf.textures[texture_i], *texture); diff --git a/impeller/scene/importer/importer_unittests.cc b/impeller/scene/importer/importer_unittests.cc index 1c98623068554..de07f76c2d843 100644 --- a/impeller/scene/importer/importer_unittests.cc +++ b/impeller/scene/importer/importer_unittests.cc @@ -74,6 +74,8 @@ TEST(ImporterTest, CanParseSkinnedGLTF) { // The skinned node contains both a skeleton and skinned mesh primitives that // reference bones in the skeleton. auto& skinned_node = scene.nodes[node->children[0]]; + ASSERT_NE(skinned_node->skin, nullptr); + ASSERT_EQ(skinned_node->mesh_primitives.size(), 2u); auto& bottom_triangle = *skinned_node->mesh_primitives[0]; ASSERT_EQ(bottom_triangle.indices->count, 3u); diff --git a/impeller/scene/importer/scene.fbs b/impeller/scene/importer/scene.fbs index bfc949ae7d334..3966e0ed395d9 100644 --- a/impeller/scene/importer/scene.fbs +++ b/impeller/scene/importer/scene.fbs @@ -188,6 +188,7 @@ table Node { table Scene { children: [int]; // Indices into `Scene`->`nodes`. + transform: Matrix; nodes: [Node]; textures: [Texture]; // Textures may be reused across different materials. animations: [Animation]; diff --git a/impeller/scene/importer/vertices_builder.cc b/impeller/scene/importer/vertices_builder.cc index 14f56dc214d54..78bde249f6b53 100644 --- a/impeller/scene/importer/vertices_builder.cc +++ b/impeller/scene/importer/vertices_builder.cc @@ -65,19 +65,6 @@ static void PassthroughAttributeWriter( } } -/// @brief A ComponentWriter which converts a Vector3 position from -/// right-handed GLTF space to left-handed Impeller space. -static void PositionAttributeWriter( - Scalar* destination, - const void* source, - const VerticesBuilder::ComponentProperties& component, - const VerticesBuilder::AttributeProperties& attribute) { - FML_DCHECK(attribute.component_count == 3); - *(destination + 0) = component.convert_proc(source, 0, true); - *(destination + 1) = component.convert_proc(source, 1, true); - *(destination + 2) = -component.convert_proc(source, 2, true); -} - /// @brief A ComponentWriter which converts four vertex indices to scalars. static void JointsAttributeWriter( Scalar* destination, @@ -96,7 +83,7 @@ std::map {.offset_bytes = offsetof(UnskinnedVerticesBuilder::Vertex, position), .size_bytes = sizeof(UnskinnedVerticesBuilder::Vertex::position), .component_count = 3, - .write_proc = PositionAttributeWriter}}, + .write_proc = PassthroughAttributeWriter}}, {VerticesBuilder::AttributeType::kNormal, {.offset_bytes = offsetof(UnskinnedVerticesBuilder::Vertex, normal), .size_bytes = sizeof(UnskinnedVerticesBuilder::Vertex::normal), diff --git a/impeller/scene/mesh.cc b/impeller/scene/mesh.cc index ff9558126285c..bc4b02da00a80 100644 --- a/impeller/scene/mesh.cc +++ b/impeller/scene/mesh.cc @@ -5,9 +5,11 @@ #include "impeller/scene/mesh.h" #include +#include #include "impeller/base/validation.h" #include "impeller/scene/material.h" +#include "impeller/scene/pipeline_key.h" #include "impeller/scene/scene_encoder.h" namespace impeller { @@ -31,8 +33,11 @@ std::vector& Mesh::GetPrimitives() { return primitives_; } -bool Mesh::Render(SceneEncoder& encoder, const Matrix& transform) const { +bool Mesh::Render(SceneEncoder& encoder, + const Matrix& transform, + const std::shared_ptr& joints) const { for (const auto& mesh : primitives_) { + mesh.geometry->SetJointsTexture(joints); SceneCommand command = { .label = "Mesh Primitive", .transform = transform, diff --git a/impeller/scene/mesh.h b/impeller/scene/mesh.h index 5684b4fa5d8dd..dc33caf27b1c3 100644 --- a/impeller/scene/mesh.h +++ b/impeller/scene/mesh.h @@ -15,6 +15,8 @@ namespace impeller { namespace scene { +class Skin; + class Mesh final { public: struct Primitive { @@ -31,7 +33,9 @@ class Mesh final { void AddPrimitive(Primitive mesh_); std::vector& GetPrimitives(); - bool Render(SceneEncoder& encoder, const Matrix& transform) const; + bool Render(SceneEncoder& encoder, + const Matrix& transform, + const std::shared_ptr& joints) const; private: std::vector primitives_; diff --git a/impeller/scene/node.cc b/impeller/scene/node.cc index 72a551af71278..dcc9b2c213278 100644 --- a/impeller/scene/node.cc +++ b/impeller/scene/node.cc @@ -126,6 +126,8 @@ std::shared_ptr Node::MakeFromFlatbuffer(const fb::Scene& scene, } auto result = std::make_shared(); + result->SetLocalTransform(importer::ToMatrix(*scene.transform())); + if (!scene.nodes() || !scene.children()) { return result; // The scene is empty. } @@ -190,17 +192,21 @@ void Node::UnpackFromFlatbuffer( /// Child nodes. - if (!source_node.children()) { - return; + if (source_node.children()) { + // Wire up graph connections. + for (int child : *source_node.children()) { + if (child < 0 || static_cast(child) >= scene_nodes.size()) { + VALIDATION_LOG << "Node child index out of range."; + continue; + } + AddChild(scene_nodes[child]); + } } - // Wire up graph connections. - for (int child : *source_node.children()) { - if (child < 0 || static_cast(child) >= scene_nodes.size()) { - VALIDATION_LOG << "Node child index out of range."; - continue; - } - AddChild(scene_nodes[child]); + /// Skin. + + if (source_node.skin()) { + skin_ = Skin::MakeFromFlatbuffer(*source_node.skin(), scene_nodes); } } @@ -220,6 +226,10 @@ void Node::SetName(const std::string& new_name) { name_ = new_name; } +Node* Node::GetParent() const { + return parent_; +} + std::shared_ptr Node::FindChildByName( const std::string& name, bool exclude_animation_players) const { @@ -301,16 +311,27 @@ Mesh& Node::GetMesh() { return mesh_; } -bool Node::Render(SceneEncoder& encoder, const Matrix& parent_transform) const { +void Node::SetIsJoint(bool is_joint) { + is_joint_ = is_joint; +} + +bool Node::IsJoint() const { + return is_joint_; +} + +bool Node::Render(SceneEncoder& encoder, + Allocator& allocator, + const Matrix& parent_transform) const { if (animation_player_.has_value()) { animation_player_->Update(); } Matrix transform = parent_transform * local_transform_; - mesh_.Render(encoder, transform); + mesh_.Render(encoder, transform, + skin_ ? skin_->GetJointsTexture(allocator) : nullptr); for (auto& child : children_) { - if (!child->Render(encoder, transform)) { + if (!child->Render(encoder, allocator, transform)) { return false; } } diff --git a/impeller/scene/node.h b/impeller/scene/node.h index e573f9991b236..8b79e5f90fd81 100644 --- a/impeller/scene/node.h +++ b/impeller/scene/node.h @@ -18,6 +18,7 @@ #include "impeller/scene/camera.h" #include "impeller/scene/mesh.h" #include "impeller/scene/scene_encoder.h" +#include "impeller/scene/skin.h" namespace impeller { namespace scene { @@ -36,6 +37,8 @@ class Node final { const std::string& GetName() const; void SetName(const std::string& new_name); + Node* GetParent() const; + std::shared_ptr FindChildByName( const std::string& name, bool exclude_animation_players = false) const; @@ -55,7 +58,12 @@ class Node final { void SetMesh(Mesh mesh); Mesh& GetMesh(); - bool Render(SceneEncoder& encoder, const Matrix& parent_transform) const; + void SetIsJoint(bool is_joint); + bool IsJoint() const; + + bool Render(SceneEncoder& encoder, + Allocator& allocator, + const Matrix& parent_transform) const; private: void UnpackFromFlatbuffer( @@ -68,6 +76,7 @@ class Node final { std::string name_; bool is_root_ = false; + bool is_joint_ = false; Node* parent_ = nullptr; std::vector> children_; Mesh mesh_; @@ -76,6 +85,8 @@ class Node final { std::vector> animations_; mutable std::optional animation_player_; + std::unique_ptr skin_; + FML_DISALLOW_COPY_AND_ASSIGN(Node); friend Scene; diff --git a/impeller/scene/scene.cc b/impeller/scene/scene.cc index cf88faa10cfc4..d1c8cae5002b2 100644 --- a/impeller/scene/scene.cc +++ b/impeller/scene/scene.cc @@ -34,7 +34,9 @@ bool Scene::Render(const RenderTarget& render_target, const Matrix& camera_transform) const { // Collect the render commands from the scene. SceneEncoder encoder; - if (!root_.Render(encoder, Matrix())) { + if (!root_.Render(encoder, + *scene_context_->GetContext()->GetResourceAllocator(), + Matrix())) { FML_LOG(ERROR) << "Failed to render frame."; return false; } diff --git a/impeller/scene/scene_context.cc b/impeller/scene/scene_context.cc index 3f1ab5ba6cd85..284d6da6b3ac1 100644 --- a/impeller/scene/scene_context.cc +++ b/impeller/scene/scene_context.cc @@ -51,14 +51,15 @@ SceneContext::SceneContext(std::shared_ptr context) placeholder_texture_ = context_->GetResourceAllocator()->CreateTexture(texture_descriptor); + placeholder_texture_->SetLabel("Placeholder Texture"); if (!placeholder_texture_) { - FML_DLOG(ERROR) << "Could not create placeholder texture."; + FML_LOG(ERROR) << "Could not create placeholder texture."; return; } uint8_t pixel[] = {0xFF, 0xFF, 0xFF, 0xFF}; - if (!placeholder_texture_->SetContents(pixel, 4, 0)) { - FML_DLOG(ERROR) << "Could not set contents of placeholder texture."; + if (!placeholder_texture_->SetContents(pixel, 4)) { + FML_LOG(ERROR) << "Could not set contents of placeholder texture."; return; } } diff --git a/impeller/scene/scene_unittests.cc b/impeller/scene/scene_unittests.cc index 5ce513ab165e2..f2a048f8a7f4f 100644 --- a/impeller/scene/scene_unittests.cc +++ b/impeller/scene/scene_unittests.cc @@ -135,6 +135,31 @@ TEST_P(SceneTest, TwoTriangles) { scene.GetRoot().AddChild(std::move(gltf_scene)); Renderer::RenderCallback callback = [&](RenderTarget& render_target) { + ImGui::Begin("Controls", nullptr, ImGuiWindowFlags_AlwaysAutoResize); + { + static Scalar playback_time_scale = 1; + static Scalar weight = 1; + static bool loop = true; + + ImGui::SliderFloat("Playback time scale", &playback_time_scale, -5, 5); + ImGui::SliderFloat("Weight", &weight, -2, 2); + ImGui::Checkbox("Loop", &loop); + if (ImGui::Button("Play")) { + metronome_clip.Play(); + } + if (ImGui::Button("Pause")) { + metronome_clip.Pause(); + } + if (ImGui::Button("Stop")) { + metronome_clip.Stop(); + } + + metronome_clip.SetPlaybackTimeScale(playback_time_scale); + metronome_clip.SetWeight(weight); + metronome_clip.SetLoop(loop); + } + + ImGui::End(); Node& node = *scene.GetRoot().GetChildren()[0]; node.SetLocalTransform(node.GetLocalTransform() * Matrix::MakeRotation(0.02, {0, 1, 0, 0})); diff --git a/impeller/scene/shaders/skinned.vert b/impeller/scene/shaders/skinned.vert index 887ce8f1f3778..884b204d796ba 100644 --- a/impeller/scene/shaders/skinned.vert +++ b/impeller/scene/shaders/skinned.vert @@ -4,9 +4,13 @@ uniform VertInfo { mat4 mvp; + float enable_skinning; + float joint_texture_size; } vert_info; +uniform sampler2D joints_texture; + // This attribute layout is expected to be identical to `SkinnedVertex` within // `impeller/scene/importer/scene.fbs`. in vec3 position; @@ -14,7 +18,6 @@ in vec3 normal; in vec4 tangent; in vec2 texture_coords; in vec4 color; -// TODO(bdero): Use the joint indices to sample bone matrices from a texture. in vec4 joints; in vec4 weights; @@ -23,18 +26,50 @@ out mat3 v_tangent_space; out vec2 v_texture_coords; out vec4 v_color; +const int kMatrixTexelStride = 4; + +mat4 GetJoint(float joint_index) { + // The size of one texel in UV space. The joint texture should always be + // square, so the answer is the same in both dimensions. + float texel_size_uv = 1 / vert_info.joint_texture_size; + + // Each joint matrix takes up 4 pixels (16 floats), so we jump 4 pixels per + // joint matrix. + float matrix_start = joint_index * kMatrixTexelStride; + + // The texture space coordinates at the start of the matrix. + float x = mod(matrix_start, vert_info.joint_texture_size); + float y = floor(matrix_start / vert_info.joint_texture_size); + + // Nearest sample the middle of each the texel by adding `0.5 * texel_size_uv` + // to both dimensions. + y = (y + 0.5) * texel_size_uv; + mat4 joint = + mat4(texture(joints_texture, vec2((x + 0.5) * texel_size_uv, y)), + texture(joints_texture, vec2((x + 1.5) * texel_size_uv, y)), + texture(joints_texture, vec2((x + 2.5) * texel_size_uv, y)), + texture(joints_texture, vec2((x + 3.5) * texel_size_uv, y))); + + return joint; +} + void main() { - // The following two lines are temporary placeholders to prevent the vertex - // attributes from being removed from the shader. - v_color = joints; - v_color = weights; + mat4 skin_matrix; + if (vert_info.enable_skinning == 1) { + skin_matrix = + GetJoint(joints.x) * weights.x + GetJoint(joints.y) * weights.y + + GetJoint(joints.z) * weights.z + GetJoint(joints.w) * weights.w; + } else { + skin_matrix = mat4(1); // Identity matrix. + } - gl_Position = vert_info.mvp * vec4(position, 1.0); + gl_Position = vert_info.mvp * skin_matrix * vec4(position, 1.0); v_position = gl_Position.xyz; - vec3 lh_tangent = tangent.xyz * tangent.w; - v_tangent_space = - mat3(vert_info.mvp) * mat3(lh_tangent, cross(normal, lh_tangent), normal); + vec3 lh_tangent = (skin_matrix * vec4(tangent.xyz * tangent.w, 0.0)).xyz; + vec3 out_normal = (skin_matrix * vec4(normal, 0.0)).xyz; + v_tangent_space = mat3(vert_info.mvp) * + mat3(lh_tangent, cross(out_normal, lh_tangent), out_normal); v_texture_coords = texture_coords; v_color = color; } diff --git a/impeller/scene/skin.cc b/impeller/scene/skin.cc new file mode 100644 index 0000000000000..4a4c614195289 --- /dev/null +++ b/impeller/scene/skin.cc @@ -0,0 +1,121 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/scene/skin.h" + +#include +#include +#include +#include + +#include "flutter/fml/logging.h" +#include "impeller/renderer/allocator.h" +#include "impeller/scene/importer/conversions.h" + +namespace impeller { +namespace scene { + +std::unique_ptr Skin::MakeFromFlatbuffer( + const fb::Skin& skin, + const std::vector>& scene_nodes) { + if (!skin.joints() || !skin.inverse_bind_matrices() || + skin.joints()->size() != skin.inverse_bind_matrices()->size()) { + VALIDATION_LOG << "Skin data is missing joints or bind matrices."; + return nullptr; + } + + Skin result; + + result.joints_.reserve(skin.joints()->size()); + for (auto joint : *skin.joints()) { + if (joint < 0 || static_cast(joint) > scene_nodes.size()) { + VALIDATION_LOG << "Skin joint index out of range."; + result.joints_.push_back(nullptr); + continue; + } + if (scene_nodes[joint]) { + scene_nodes[joint]->SetIsJoint(true); + } + result.joints_.push_back(scene_nodes[joint]); + } + + result.inverse_bind_matrices_.reserve(skin.inverse_bind_matrices()->size()); + for (auto matrix : *skin.inverse_bind_matrices()) { + if (!matrix) { + result.inverse_bind_matrices_.push_back(Matrix()); + continue; + } + result.inverse_bind_matrices_.push_back(importer::ToMatrix(*matrix)); + } + + return std::make_unique(std::move(result)); +} + +Skin::Skin() = default; + +Skin::~Skin() = default; + +Skin::Skin(Skin&&) = default; + +Skin& Skin::operator=(Skin&&) = default; + +std::shared_ptr Skin::GetJointsTexture(Allocator& allocator) { + // Each joint has a matrix. 1 matrix = 16 floats. 1 pixel = 4 floats. + // Therefore, each joint needs 4 pixels. + auto required_pixels = joints_.size() * 4; + auto dimension_size = std::max( + 2u, + Allocation::NextPowerOfTwoSize(std::ceil(std::sqrt(required_pixels)))); + + impeller::TextureDescriptor texture_descriptor; + texture_descriptor.storage_mode = impeller::StorageMode::kHostVisible; + texture_descriptor.format = PixelFormat::kR32G32B32A32Float; + texture_descriptor.size = {dimension_size, dimension_size}; + texture_descriptor.mip_count = 1u; + + auto result = allocator.CreateTexture(texture_descriptor); + result->SetLabel("Joints Texture"); + if (!result) { + FML_LOG(ERROR) << "Could not create joint texture."; + return nullptr; + } + + std::vector joints; + joints.resize(result->GetSize().Area() / 4, Matrix()); + FML_DCHECK(joints.size() >= joints_.size()); + for (size_t joint_i = 0; joint_i < joints_.size(); joint_i++) { + const Node* joint = joints_[joint_i].get(); + if (!joint) { + // When a joint is missing, just let it remain as an identity matrix. + continue; + } + + // Compute a model space matrix for the joint by walking up the bones to the + // skeleton root. + while (joint && joint->IsJoint()) { + joints[joint_i] = joint->GetLocalTransform() * joints[joint_i]; + joint = joint->GetParent(); + } + + // Get the joint transform relative to the default pose of the bone by + // incorporating the joint's inverse bind matrix. The inverse bind matrix + // transforms from model space to the default pose space of the joint. The + // result is a model space matrix that only captures the difference between + // the joint's default pose and the joint's current pose in the scene. This + // is necessary because the skinned model's vertex positions (which _define_ + // the default pose) are all in model space. + joints[joint_i] = joints[joint_i] * inverse_bind_matrices_[joint_i]; + } + + if (!result->SetContents(reinterpret_cast(joints.data()), + joints.size() * sizeof(Matrix))) { + FML_LOG(ERROR) << "Could not set contents of joint texture."; + return nullptr; + } + + return result; +} + +} // namespace scene +} // namespace impeller diff --git a/impeller/scene/skin.h b/impeller/scene/skin.h new file mode 100644 index 0000000000000..016b527e771c0 --- /dev/null +++ b/impeller/scene/skin.h @@ -0,0 +1,42 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include +#include + +#include "flutter/fml/macros.h" + +#include "impeller/renderer/allocator.h" +#include "impeller/renderer/texture.h" +#include "impeller/scene/importer/scene_flatbuffers.h" +#include "impeller/scene/node.h" + +namespace impeller { +namespace scene { + +class Skin final { + public: + static std::unique_ptr MakeFromFlatbuffer( + const fb::Skin& skin, + const std::vector>& scene_nodes); + ~Skin(); + + Skin(Skin&&); + Skin& operator=(Skin&&); + + std::shared_ptr GetJointsTexture(Allocator& allocator); + + private: + Skin(); + + std::vector> joints_; + std::vector inverse_bind_matrices_; + + FML_DISALLOW_COPY_AND_ASSIGN(Skin); +}; + +} // namespace scene +} // namespace impeller