From fff0ab8d5b331fabb93a382e5a5b5addbd76c9db Mon Sep 17 00:00:00 2001 From: Jim Graham Date: Mon, 16 Mar 2020 13:34:12 -0700 Subject: [PATCH 1/8] Enhance image_filter_layer caching to filter a cached child --- flow/layers/container_layer.cc | 33 ++++++++++++++++ flow/layers/container_layer.h | 65 +++++++++++++++++++++++++++++-- flow/layers/image_filter_layer.cc | 30 +++++++++++--- flow/layers/image_filter_layer.h | 8 +++- flow/layers/opacity_layer.cc | 30 +------------- flow/layers/opacity_layer.h | 56 +------------------------- flow/raster_cache.cc | 1 + 7 files changed, 130 insertions(+), 93 deletions(-) diff --git a/flow/layers/container_layer.cc b/flow/layers/container_layer.cc index 2e1f4cc103eb0..7218cfa9e68cf 100644 --- a/flow/layers/container_layer.cc +++ b/flow/layers/container_layer.cc @@ -160,4 +160,37 @@ void ContainerLayer::UpdateSceneChildren(SceneUpdateContext& context) { #endif // defined(OS_FUCHSIA) +MergedContainerLayer::MergedContainerLayer() { + // Ensure the layer has only one direct child. + // + // This intermediary container helps container layers that want or need + // to cache the rendering of their children to do so more easily. + // + // Any children will be actually added as children of this empty + // ContainerLayer. + // + // @see GetChildContainer() + // @see GetCacheableChild() + ContainerLayer::Add(std::make_shared()); +} + +void MergedContainerLayer::Add(std::shared_ptr layer) { + GetChildContainer()->Add(std::move(layer)); +} + +ContainerLayer* MergedContainerLayer::GetChildContainer() const { + FML_DCHECK(layers().size() == 1); + + return static_cast(layers()[0].get()); +} + +Layer* MergedContainerLayer::GetCacheableChild() const { + ContainerLayer* child_container = GetChildContainer(); + if (child_container->layers().size() == 1) { + return child_container->layers()[0].get(); + } + + return child_container; +} + } // namespace flutter diff --git a/flow/layers/container_layer.h b/flow/layers/container_layer.h index 6b30f389c9aa1..6c26bcaba62b6 100644 --- a/flow/layers/container_layer.h +++ b/flow/layers/container_layer.h @@ -35,9 +35,6 @@ class ContainerLayer : public Layer { void UpdateSceneChildren(SceneUpdateContext& context); #endif // defined(OS_FUCHSIA) - // For OpacityLayer to restructure to have a single child. - void ClearChildren() { layers_.clear(); } - // Try to prepare the raster cache for a given layer. // // The raster cache would fail if either of the followings is true: @@ -58,6 +55,68 @@ class ContainerLayer : public Layer { FML_DISALLOW_COPY_AND_ASSIGN(ContainerLayer); }; +class MergedContainerLayer : public ContainerLayer { + public: + MergedContainerLayer(); + + void Add(std::shared_ptr layer) override; + + protected: + /** + * @brief Returns the ContainerLayer used to hold all of the children + * of the OpacityLayer. + * + * Often opacity layers will only have a single child since the associated + * Flutter widget is specified with only a single child widget pointer. + * But depending on the structure of the child tree that single widget at + * the framework level can turn into multiple children at the engine + * API level since there is no guarantee of a 1:1 correspondence of widgets + * to engine layers. This synthetic child container layer is established to + * hold all of the children in a single layer so that we can cache their + * output, but this synthetic layer will typically not be the best choice + * for the layer cache since the synthetic container is created fresh with + * each new OpacityLayer, and so may not be stable from frame to frame. + * + * @see GetCacheableChild() + * @return the ContainerLayer child used to hold the children + */ + ContainerLayer* GetChildContainer() const; + + /** + * @brief Returns the best choice for a Layer object that can be used + * in RasterCache operations to cache the children of the OpacityLayer. + * + * The returned Layer must represent all children and try to remain stable + * if the OpacityLayer is reconstructed in subsequent frames of the scene. + * + * Note that since the synthetic child container returned from the + * GetChildContainer() method is created fresh with each new OpacityLayer, + * its return value will not be a good candidate for caching. But if the + * standard recommendations for animations are followed and the child widget + * is wrapped with a RepaintBoundary widget at the framework level, then + * the synthetic child container should contain the same single child layer + * on each frame. Under those conditions, that single child of the child + * container will be the best candidate for caching in the RasterCache + * and this method will return that single child if possible to improve + * the performance of caching the children. + * + * Note that if GetCacheableChild() does not find a single stable child of + * the child container it will return the child container as a fallback. + * Even though that child is new in each frame of an animation and thus we + * cannot reuse the cached layer raster between animation frames, the single + * container child will allow us to paint the child onto an offscreen buffer + * during Preroll() which reduces one render target switch compared to + * painting the child on the fly via an AutoSaveLayer in Paint() and thus + * still improves our performance. + * + * @see GetChildContainer() + * @return the best candidate Layer for caching the children + */ + Layer* GetCacheableChild() const; + + FML_DISALLOW_COPY_AND_ASSIGN(MergedContainerLayer); +}; + } // namespace flutter #endif // FLUTTER_FLOW_LAYERS_CONTAINER_LAYER_H_ diff --git a/flow/layers/image_filter_layer.cc b/flow/layers/image_filter_layer.cc index 3a1ebbc1d619d..2c6cadc33e146 100644 --- a/flow/layers/image_filter_layer.cc +++ b/flow/layers/image_filter_layer.cc @@ -7,7 +7,7 @@ namespace flutter { ImageFilterLayer::ImageFilterLayer(sk_sp filter) - : filter_(std::move(filter)) {} + : filter_(std::move(filter)), render_count_(1) {} void ImageFilterLayer::Preroll(PrerollContext* context, const SkMatrix& matrix) { @@ -28,18 +28,38 @@ void ImageFilterLayer::Preroll(PrerollContext* context, set_paint_bounds(child_paint_bounds_); } - TryToPrepareRasterCache(context, this, matrix); + if (render_count_ >= kMinimumRendersBeforeCachingFilterLayer) { + TryToPrepareRasterCache(context, this, matrix); + } else { + render_count_++; + context->raster_cache->Prepare(context, GetCacheableChild(), matrix); + } } void ImageFilterLayer::Paint(PaintContext& context) const { TRACE_EVENT0("flutter", "ImageFilterLayer::Paint"); FML_DCHECK(needs_painting()); - if (context.raster_cache && - context.raster_cache->Draw(this, *context.leaf_nodes_canvas)) { - return; + if (context.raster_cache) { + if (context.raster_cache->Draw(this, *context.leaf_nodes_canvas)) { + FML_LOG(ERROR) << "Rendered filtered output from cache"; + return; + } + const SkMatrix& ctm = context.leaf_nodes_canvas->getTotalMatrix(); + sk_sp transformed_filter = filter_->makeWithLocalMatrix(ctm); + if (transformed_filter) { + SkPaint paint; + paint.setImageFilter(transformed_filter); + + if (context.raster_cache->Draw(GetCacheableChild(), + *context.leaf_nodes_canvas, &paint)) { + FML_LOG(ERROR) << "Filtered from cached child"; + return; + } + } } + FML_LOG(ERROR) << "Applying filter to child on the fly"; SkPaint paint; paint.setImageFilter(filter_); diff --git a/flow/layers/image_filter_layer.h b/flow/layers/image_filter_layer.h index 8e40a9ab339ba..a547833b458fd 100644 --- a/flow/layers/image_filter_layer.h +++ b/flow/layers/image_filter_layer.h @@ -11,7 +11,7 @@ namespace flutter { -class ImageFilterLayer : public ContainerLayer { +class ImageFilterLayer : public MergedContainerLayer { public: ImageFilterLayer(sk_sp filter); @@ -20,8 +20,14 @@ class ImageFilterLayer : public ContainerLayer { void Paint(PaintContext& context) const override; private: + // The default minimum number of times to render a filtered layer before + // we cache the output of the filter. Note that until this limit is + // reached we may continue to cache the children anyway. + static constexpr int kMinimumRendersBeforeCachingFilterLayer = 3; + sk_sp filter_; SkRect child_paint_bounds_; + int render_count_; FML_DISALLOW_COPY_AND_ASSIGN(ImageFilterLayer); }; diff --git a/flow/layers/opacity_layer.cc b/flow/layers/opacity_layer.cc index 0d9ade296ddfd..0696d11fa3b60 100644 --- a/flow/layers/opacity_layer.cc +++ b/flow/layers/opacity_layer.cc @@ -10,20 +10,7 @@ namespace flutter { OpacityLayer::OpacityLayer(SkAlpha alpha, const SkPoint& offset) - : alpha_(alpha), offset_(offset) { - // Ensure OpacityLayer has only one direct child. - // - // This is needed to ensure that retained rendering can always be applied to - // save the costly saveLayer. - // - // Any children will be actually added as children of this empty - // ContainerLayer. - ContainerLayer::Add(std::make_shared()); -} - -void OpacityLayer::Add(std::shared_ptr layer) { - GetChildContainer()->Add(std::move(layer)); -} + : alpha_(alpha), offset_(offset) {} void OpacityLayer::Preroll(PrerollContext* context, const SkMatrix& matrix) { TRACE_EVENT0("flutter", "OpacityLayer::Preroll"); @@ -112,19 +99,4 @@ void OpacityLayer::UpdateScene(SceneUpdateContext& context) { #endif // defined(OS_FUCHSIA) -ContainerLayer* OpacityLayer::GetChildContainer() const { - FML_DCHECK(layers().size() == 1); - - return static_cast(layers()[0].get()); -} - -Layer* OpacityLayer::GetCacheableChild() const { - ContainerLayer* child_container = GetChildContainer(); - if (child_container->layers().size() == 1) { - return child_container->layers()[0].get(); - } - - return child_container; -} - } // namespace flutter diff --git a/flow/layers/opacity_layer.h b/flow/layers/opacity_layer.h index 4edc61ad9009a..632d49368aa1e 100644 --- a/flow/layers/opacity_layer.h +++ b/flow/layers/opacity_layer.h @@ -13,7 +13,7 @@ namespace flutter { // OpacityLayer is very costly due to the saveLayer call. If there's no child, // having the OpacityLayer or not has the same effect. In debug_unopt build, // |Preroll| will assert if there are no children. -class OpacityLayer : public ContainerLayer { +class OpacityLayer : public MergedContainerLayer { public: // An offset is provided here because OpacityLayer.addToScene method in the // Flutter framework can take an optional offset argument. @@ -27,8 +27,6 @@ class OpacityLayer : public ContainerLayer { // the propagation as repainting the OpacityLayer is expensive. OpacityLayer(SkAlpha alpha, const SkPoint& offset); - void Add(std::shared_ptr layer) override; - void Preroll(PrerollContext* context, const SkMatrix& matrix) override; void Paint(PaintContext& context) const override; @@ -38,58 +36,6 @@ class OpacityLayer : public ContainerLayer { #endif // defined(OS_FUCHSIA) private: - /** - * @brief Returns the ContainerLayer used to hold all of the children - * of the OpacityLayer. - * - * Often opacity layers will only have a single child since the associated - * Flutter widget is specified with only a single child widget pointer. - * But depending on the structure of the child tree that single widget at - * the framework level can turn into multiple children at the engine - * API level since there is no guarantee of a 1:1 correspondence of widgets - * to engine layers. This synthetic child container layer is established to - * hold all of the children in a single layer so that we can cache their - * output, but this synthetic layer will typically not be the best choice - * for the layer cache since the synthetic container is created fresh with - * each new OpacityLayer, and so may not be stable from frame to frame. - * - * @see GetCacheableChild() - * @return the ContainerLayer child used to hold the children - */ - ContainerLayer* GetChildContainer() const; - - /** - * @brief Returns the best choice for a Layer object that can be used - * in RasterCache operations to cache the children of the OpacityLayer. - * - * The returned Layer must represent all children and try to remain stable - * if the OpacityLayer is reconstructed in subsequent frames of the scene. - * - * Note that since the synthetic child container returned from the - * GetChildContainer() method is created fresh with each new OpacityLayer, - * its return value will not be a good candidate for caching. But if the - * standard recommendations for animations are followed and the child widget - * is wrapped with a RepaintBoundary widget at the framework level, then - * the synthetic child container should contain the same single child layer - * on each frame. Under those conditions, that single child of the child - * container will be the best candidate for caching in the RasterCache - * and this method will return that single child if possible to improve - * the performance of caching the children. - * - * Note that if GetCacheableChild() does not find a single stable child of - * the child container it will return the child container as a fallback. - * Even though that child is new in each frame of an animation and thus we - * cannot reuse the cached layer raster between animation frames, the single - * container child will allow us to paint the child onto an offscreen buffer - * during Preroll() which reduces one render target switch compared to - * painting the child on the fly via an AutoSaveLayer in Paint() and thus - * still improves our performance. - * - * @see GetChildContainer() - * @return the best candidate Layer for caching the children - */ - Layer* GetCacheableChild() const; - SkAlpha alpha_; SkPoint offset_; SkRRect frameRRect_; diff --git a/flow/raster_cache.cc b/flow/raster_cache.cc index 89fc1ebd3c092..ef89bb4d551fc 100644 --- a/flow/raster_cache.cc +++ b/flow/raster_cache.cc @@ -141,6 +141,7 @@ void RasterCache::Prepare(PrerollContext* context, entry.access_count++; entry.used_this_frame = true; if (!entry.image) { + FML_LOG(ERROR) << "Rasterizing " << layer->unique_id(); entry.image = RasterizeLayer(context, layer, ctm, checkerboard_images_); } } From d0aa1f262b87576e5889534101993db1fe098768 Mon Sep 17 00:00:00 2001 From: Jim Graham Date: Wed, 18 Mar 2020 14:46:33 -0700 Subject: [PATCH 2/8] remove debugging prints and update comments --- flow/layers/container_layer.cc | 16 ++++++++-------- flow/layers/container_layer.h | 1 + flow/layers/image_filter_layer.cc | 20 +++++++++----------- flow/layers/image_filter_layer.h | 1 - 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/flow/layers/container_layer.cc b/flow/layers/container_layer.cc index 7218cfa9e68cf..07a826e0b7c73 100644 --- a/flow/layers/container_layer.cc +++ b/flow/layers/container_layer.cc @@ -163,14 +163,13 @@ void ContainerLayer::UpdateSceneChildren(SceneUpdateContext& context) { MergedContainerLayer::MergedContainerLayer() { // Ensure the layer has only one direct child. // - // This intermediary container helps container layers that want or need - // to cache the rendering of their children to do so more easily. - // - // Any children will be actually added as children of this empty - // ContainerLayer. - // - // @see GetChildContainer() - // @see GetCacheableChild() + // Any children will actually be added as children of this empty + // ContainerLayer which can be accessed via ::GetContainerLayer(). + // If only one child is ever added to this layer then that child + // will become the layer returned from ::GetCacheableChild(). + // If multiple child layers are added, then this implicit container + // child becomes the cacheable child, but at the potential cost of + // not being as stable in the raster cache from frame to frame. ContainerLayer::Add(std::make_shared()); } @@ -190,6 +189,7 @@ Layer* MergedContainerLayer::GetCacheableChild() const { return child_container->layers()[0].get(); } + FML_LOG(INFO) << "Single child layer contains multiple children"; return child_container; } diff --git a/flow/layers/container_layer.h b/flow/layers/container_layer.h index 6c26bcaba62b6..a2f9f9ebde9a1 100644 --- a/flow/layers/container_layer.h +++ b/flow/layers/container_layer.h @@ -114,6 +114,7 @@ class MergedContainerLayer : public ContainerLayer { */ Layer* GetCacheableChild() const; + private: FML_DISALLOW_COPY_AND_ASSIGN(MergedContainerLayer); }; diff --git a/flow/layers/image_filter_layer.cc b/flow/layers/image_filter_layer.cc index 2c6cadc33e146..86a882c4d0b46 100644 --- a/flow/layers/image_filter_layer.cc +++ b/flow/layers/image_filter_layer.cc @@ -16,17 +16,16 @@ void ImageFilterLayer::Preroll(PrerollContext* context, Layer::AutoPrerollSaveLayerState save = Layer::AutoPrerollSaveLayerState::Create(context); - child_paint_bounds_ = SkRect::MakeEmpty(); - PrerollChildren(context, matrix, &child_paint_bounds_); + SkRect child_bounds = SkRect::MakeEmpty(); + PrerollChildren(context, matrix, &child_bounds); if (filter_) { - const SkIRect filter_input_bounds = child_paint_bounds_.roundOut(); + const SkIRect filter_input_bounds = child_bounds.roundOut(); SkIRect filter_output_bounds = filter_->filterBounds(filter_input_bounds, SkMatrix::I(), SkImageFilter::kForward_MapDirection); - set_paint_bounds(SkRect::Make(filter_output_bounds)); - } else { - set_paint_bounds(child_paint_bounds_); + child_bounds = SkRect::Make(filter_output_bounds); } + set_paint_bounds(child_bounds); if (render_count_ >= kMinimumRendersBeforeCachingFilterLayer) { TryToPrepareRasterCache(context, this, matrix); @@ -59,16 +58,15 @@ void ImageFilterLayer::Paint(PaintContext& context) const { } } - FML_LOG(ERROR) << "Applying filter to child on the fly"; SkPaint paint; paint.setImageFilter(filter_); // Normally a save_layer is sized to the current layer bounds, but in this // case the bounds of the child may not be the same as the filtered version - // so we use the child_paint_bounds_ which were snapshotted from the - // Preroll on the children before we adjusted them based on the filter. - Layer::AutoSaveLayer save_layer = - Layer::AutoSaveLayer::Create(context, child_paint_bounds_, &paint); + // so we use the bounds of the child container which do not include any + // modifications that the filter might apply. + Layer::AutoSaveLayer save_layer = Layer::AutoSaveLayer::Create( + context, GetChildContainer()->paint_bounds(), &paint); PaintChildren(context); } diff --git a/flow/layers/image_filter_layer.h b/flow/layers/image_filter_layer.h index a547833b458fd..02b003fd5bd8a 100644 --- a/flow/layers/image_filter_layer.h +++ b/flow/layers/image_filter_layer.h @@ -26,7 +26,6 @@ class ImageFilterLayer : public MergedContainerLayer { static constexpr int kMinimumRendersBeforeCachingFilterLayer = 3; sk_sp filter_; - SkRect child_paint_bounds_; int render_count_; FML_DISALLOW_COPY_AND_ASSIGN(ImageFilterLayer); From c9075b94fcb4eb84782ff6685e53833f1906d228 Mon Sep 17 00:00:00 2001 From: Jim Graham Date: Mon, 22 Jun 2020 14:20:50 -0700 Subject: [PATCH 3/8] remove remaining debug prints --- flow/layers/container_layer.cc | 1 - flow/layers/image_filter_layer.cc | 2 -- flow/raster_cache.cc | 1 - 3 files changed, 4 deletions(-) diff --git a/flow/layers/container_layer.cc b/flow/layers/container_layer.cc index 07a826e0b7c73..9a18775709b64 100644 --- a/flow/layers/container_layer.cc +++ b/flow/layers/container_layer.cc @@ -189,7 +189,6 @@ Layer* MergedContainerLayer::GetCacheableChild() const { return child_container->layers()[0].get(); } - FML_LOG(INFO) << "Single child layer contains multiple children"; return child_container; } diff --git a/flow/layers/image_filter_layer.cc b/flow/layers/image_filter_layer.cc index 86a882c4d0b46..61c630324dded 100644 --- a/flow/layers/image_filter_layer.cc +++ b/flow/layers/image_filter_layer.cc @@ -41,7 +41,6 @@ void ImageFilterLayer::Paint(PaintContext& context) const { if (context.raster_cache) { if (context.raster_cache->Draw(this, *context.leaf_nodes_canvas)) { - FML_LOG(ERROR) << "Rendered filtered output from cache"; return; } const SkMatrix& ctm = context.leaf_nodes_canvas->getTotalMatrix(); @@ -52,7 +51,6 @@ void ImageFilterLayer::Paint(PaintContext& context) const { if (context.raster_cache->Draw(GetCacheableChild(), *context.leaf_nodes_canvas, &paint)) { - FML_LOG(ERROR) << "Filtered from cached child"; return; } } diff --git a/flow/raster_cache.cc b/flow/raster_cache.cc index ef89bb4d551fc..89fc1ebd3c092 100644 --- a/flow/raster_cache.cc +++ b/flow/raster_cache.cc @@ -141,7 +141,6 @@ void RasterCache::Prepare(PrerollContext* context, entry.access_count++; entry.used_this_frame = true; if (!entry.image) { - FML_LOG(ERROR) << "Rasterizing " << layer->unique_id(); entry.image = RasterizeLayer(context, layer, ctm, checkerboard_images_); } } From a914f67cc2c21b6d759b1d60b6fd842744304530 Mon Sep 17 00:00:00 2001 From: Jim Graham Date: Mon, 22 Jun 2020 15:18:51 -0700 Subject: [PATCH 4/8] fix unit tests and add class doc comment for MergedContainerLayer --- flow/layers/container_layer.h | 31 +++++++++++++++++++++++++++++++ flow/layers/image_filter_layer.cc | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/flow/layers/container_layer.h b/flow/layers/container_layer.h index a2f9f9ebde9a1..8ce416e497ad9 100644 --- a/flow/layers/container_layer.h +++ b/flow/layers/container_layer.h @@ -55,6 +55,37 @@ class ContainerLayer : public Layer { FML_DISALLOW_COPY_AND_ASSIGN(ContainerLayer); }; +//------------------------------------------------------------------------------ +/// Some ContainerLayer objects need to cache their children. The existing +/// layer caching mechanism only supports caching an individual layer as it +/// uses the unique_id() of that layer as the cache key. If a ContainerLayer +/// has multiple children and needs to cache them then it will need to first +/// collect them into a single layer object to satisfy the caching mechanism. +/// This utility class will silently collect all of the children of a +/// ContainerLayer into a single merged ContainerLayer automatically. +/// +/// The Flutter Widget objects that produce these layers tend to enforce +/// having only a single child Widget object which means that there is really +/// only a single child being added to this engine layer anyway, but due to +/// the complicated mechanism that turns trees of Widgets eventually into a +/// tree of engine layers, the associated layer may end up with multiple +/// direct child layers. This class implements a protection against the +/// incidental creation of "extra" children. +/// +/// When the layer needs to recurse to perform some opertion on its children, +/// it can call GetChildContainer() to return the hidden container containing +/// all of the real children. +/// +/// When the layer wants to cache the rendered contents of its children, it +/// should call GetCacheableChild() for best performance. +/// +/// Note that since the implicit child container is created new every time +/// this layer is recreated, the child container will be different for each +/// frame of an animation that modifies the properties of this layer. +/// The GetCacheableChild() method will attempt to find the single actual +/// child of this layer and return that instead since that child will more +/// likely remain stable across the frames of an animation. +/// class MergedContainerLayer : public ContainerLayer { public: MergedContainerLayer(); diff --git a/flow/layers/image_filter_layer.cc b/flow/layers/image_filter_layer.cc index 61c630324dded..448e81897aa8b 100644 --- a/flow/layers/image_filter_layer.cc +++ b/flow/layers/image_filter_layer.cc @@ -31,7 +31,7 @@ void ImageFilterLayer::Preroll(PrerollContext* context, TryToPrepareRasterCache(context, this, matrix); } else { render_count_++; - context->raster_cache->Prepare(context, GetCacheableChild(), matrix); + TryToPrepareRasterCache(context, GetCacheableChild(), matrix); } } From 98cdf3eafe5206c16825890091c6d1d63971619a Mon Sep 17 00:00:00 2001 From: Jim Graham Date: Mon, 22 Jun 2020 15:50:49 -0700 Subject: [PATCH 5/8] add unit tests for MergedContainerLayer and ImageFilterLayer caching --- flow/layers/container_layer_unittests.cc | 70 +++++++++++++++++++++ flow/layers/image_filter_layer_unittests.cc | 67 ++++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/flow/layers/container_layer_unittests.cc b/flow/layers/container_layer_unittests.cc index 1c11e7c02b88d..65c2ff365fa0c 100644 --- a/flow/layers/container_layer_unittests.cc +++ b/flow/layers/container_layer_unittests.cc @@ -198,5 +198,75 @@ TEST_F(ContainerLayerTest, NeedsSystemComposite) { child_path2, child_paint2}}})); } +TEST_F(ContainerLayerTest, MergedOneChild) { + SkPath child_path; + child_path.addRect(5.0f, 6.0f, 20.5f, 21.5f); + SkPaint child_paint(SkColors::kGreen); + SkMatrix initial_transform = SkMatrix::Translate(-0.5f, -0.5f); + + auto mock_layer = std::make_shared(child_path, child_paint); + auto layer = std::make_shared(); + layer->Add(mock_layer); + + layer->Preroll(preroll_context(), initial_transform); + EXPECT_FALSE(preroll_context()->has_platform_view); + EXPECT_EQ(mock_layer->paint_bounds(), child_path.getBounds()); + EXPECT_EQ(layer->paint_bounds(), child_path.getBounds()); + EXPECT_TRUE(mock_layer->needs_painting()); + EXPECT_TRUE(layer->needs_painting()); + EXPECT_FALSE(mock_layer->needs_system_composite()); + EXPECT_FALSE(layer->needs_system_composite()); + EXPECT_EQ(mock_layer->parent_matrix(), initial_transform); + EXPECT_EQ(mock_layer->parent_cull_rect(), kGiantRect); + + layer->Paint(paint_context()); + EXPECT_EQ(mock_canvas().draw_calls(), + std::vector({MockCanvas::DrawCall{ + 0, MockCanvas::DrawPathData{child_path, child_paint}}})); +} + +TEST_F(ContainerLayerTest, MergedMultipleChildren) { + SkPath child_path1; + child_path1.addRect(5.0f, 6.0f, 20.5f, 21.5f); + SkPath child_path2; + child_path2.addRect(58.0f, 2.0f, 16.5f, 14.5f); + SkPaint child_paint1(SkColors::kGray); + SkPaint child_paint2(SkColors::kGreen); + SkMatrix initial_transform = SkMatrix::Translate(-0.5f, -0.5f); + + auto mock_layer1 = std::make_shared(child_path1, child_paint1); + auto mock_layer2 = std::make_shared(child_path2, child_paint2); + auto layer = std::make_shared(); + layer->Add(mock_layer1); + layer->Add(mock_layer2); + + SkRect expected_total_bounds = child_path1.getBounds(); + expected_total_bounds.join(child_path2.getBounds()); + layer->Preroll(preroll_context(), initial_transform); + EXPECT_FALSE(preroll_context()->has_platform_view); + EXPECT_EQ(mock_layer1->paint_bounds(), child_path1.getBounds()); + EXPECT_EQ(mock_layer2->paint_bounds(), child_path2.getBounds()); + EXPECT_EQ(layer->paint_bounds(), expected_total_bounds); + EXPECT_TRUE(mock_layer1->needs_painting()); + EXPECT_TRUE(mock_layer2->needs_painting()); + EXPECT_TRUE(layer->needs_painting()); + EXPECT_FALSE(mock_layer1->needs_system_composite()); + EXPECT_FALSE(mock_layer2->needs_system_composite()); + EXPECT_FALSE(layer->needs_system_composite()); + EXPECT_EQ(mock_layer1->parent_matrix(), initial_transform); + EXPECT_EQ(mock_layer2->parent_matrix(), initial_transform); + EXPECT_EQ(mock_layer1->parent_cull_rect(), kGiantRect); + EXPECT_EQ(mock_layer2->parent_cull_rect(), + kGiantRect); // Siblings are independent + + layer->Paint(paint_context()); + EXPECT_EQ( + mock_canvas().draw_calls(), + std::vector({MockCanvas::DrawCall{ + 0, MockCanvas::DrawPathData{child_path1, child_paint1}}, + MockCanvas::DrawCall{0, MockCanvas::DrawPathData{ + child_path2, child_paint2}}})); +} + } // namespace testing } // namespace flutter diff --git a/flow/layers/image_filter_layer_unittests.cc b/flow/layers/image_filter_layer_unittests.cc index 3583d1540dbf6..6616e648e4265 100644 --- a/flow/layers/image_filter_layer_unittests.cc +++ b/flow/layers/image_filter_layer_unittests.cc @@ -260,5 +260,72 @@ TEST_F(ImageFilterLayerTest, Readback) { EXPECT_FALSE(preroll_context()->surface_needs_readback); } +TEST_F(ImageFilterLayerTest, ChildIsCached) { + auto layer_filter = SkImageFilter::MakeMatrixFilter( + SkMatrix(), SkFilterQuality::kMedium_SkFilterQuality, nullptr); + auto initial_transform = SkMatrix::Translate(50.0, 25.5); + auto other_transform = SkMatrix::Scale(1.0, 2.0); + const SkPath child_path = SkPath().addRect(SkRect::MakeWH(5.0f, 5.0f)); + auto mock_layer = std::make_shared(child_path); + auto layer = + std::make_shared(layer_filter); + layer->Add(mock_layer); + + SkMatrix cache_ctm = initial_transform; + SkCanvas cache_canvas; + cache_canvas.setMatrix(cache_ctm); + SkCanvas other_canvas; + other_canvas.setMatrix(other_transform); + + use_mock_raster_cache(); + + EXPECT_EQ(raster_cache()->GetLayerCachedEntriesCount(), (size_t)0); + EXPECT_FALSE(raster_cache()->Draw(mock_layer.get(), other_canvas)); + EXPECT_FALSE(raster_cache()->Draw(mock_layer.get(), cache_canvas)); + + layer->Preroll(preroll_context(), initial_transform); + + EXPECT_EQ(raster_cache()->GetLayerCachedEntriesCount(), (size_t)1); + EXPECT_FALSE(raster_cache()->Draw(mock_layer.get(), other_canvas)); + EXPECT_TRUE(raster_cache()->Draw(mock_layer.get(), cache_canvas)); +} + +TEST_F(ImageFilterLayerTest, ChildrenNotCached) { + auto layer_filter = SkImageFilter::MakeMatrixFilter( + SkMatrix(), SkFilterQuality::kMedium_SkFilterQuality, nullptr); + auto initial_transform = SkMatrix::Translate(50.0, 25.5); + auto other_transform = SkMatrix::Scale(1.0, 2.0); + const SkPath child_path1 = SkPath().addRect(SkRect::MakeWH(5.0f, 5.0f)); + const SkPath child_path2 = SkPath().addRect(SkRect::MakeWH(5.0f, 5.0f)); + auto mock_layer1 = std::make_shared(child_path1); + auto mock_layer2 = std::make_shared(child_path2); + auto layer = + std::make_shared(layer_filter); + layer->Add(mock_layer1); + layer->Add(mock_layer2); + + SkMatrix cache_ctm = initial_transform; + SkCanvas cache_canvas; + cache_canvas.setMatrix(cache_ctm); + SkCanvas other_canvas; + other_canvas.setMatrix(other_transform); + + use_mock_raster_cache(); + + EXPECT_EQ(raster_cache()->GetLayerCachedEntriesCount(), (size_t)0); + EXPECT_FALSE(raster_cache()->Draw(mock_layer1.get(), other_canvas)); + EXPECT_FALSE(raster_cache()->Draw(mock_layer1.get(), cache_canvas)); + EXPECT_FALSE(raster_cache()->Draw(mock_layer2.get(), other_canvas)); + EXPECT_FALSE(raster_cache()->Draw(mock_layer2.get(), cache_canvas)); + + layer->Preroll(preroll_context(), initial_transform); + + EXPECT_EQ(raster_cache()->GetLayerCachedEntriesCount(), (size_t)1); + EXPECT_FALSE(raster_cache()->Draw(mock_layer1.get(), other_canvas)); + EXPECT_FALSE(raster_cache()->Draw(mock_layer1.get(), cache_canvas)); + EXPECT_FALSE(raster_cache()->Draw(mock_layer2.get(), other_canvas)); + EXPECT_FALSE(raster_cache()->Draw(mock_layer2.get(), cache_canvas)); +} + } // namespace testing } // namespace flutter From 25f98e339f851e89a570007f38e437f14df55853 Mon Sep 17 00:00:00 2001 From: Jim Graham Date: Mon, 22 Jun 2020 19:19:50 -0700 Subject: [PATCH 6/8] fixed formatting --- flow/layers/image_filter_layer_unittests.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/flow/layers/image_filter_layer_unittests.cc b/flow/layers/image_filter_layer_unittests.cc index 6616e648e4265..bcc2d8e08e8b0 100644 --- a/flow/layers/image_filter_layer_unittests.cc +++ b/flow/layers/image_filter_layer_unittests.cc @@ -267,8 +267,7 @@ TEST_F(ImageFilterLayerTest, ChildIsCached) { auto other_transform = SkMatrix::Scale(1.0, 2.0); const SkPath child_path = SkPath().addRect(SkRect::MakeWH(5.0f, 5.0f)); auto mock_layer = std::make_shared(child_path); - auto layer = - std::make_shared(layer_filter); + auto layer = std::make_shared(layer_filter); layer->Add(mock_layer); SkMatrix cache_ctm = initial_transform; @@ -299,8 +298,7 @@ TEST_F(ImageFilterLayerTest, ChildrenNotCached) { const SkPath child_path2 = SkPath().addRect(SkRect::MakeWH(5.0f, 5.0f)); auto mock_layer1 = std::make_shared(child_path1); auto mock_layer2 = std::make_shared(child_path2); - auto layer = - std::make_shared(layer_filter); + auto layer = std::make_shared(layer_filter); layer->Add(mock_layer1); layer->Add(mock_layer2); From f6db62a61e3658f1c9f3df37ef43c75f46da1961 Mon Sep 17 00:00:00 2001 From: Jim Graham Date: Wed, 24 Jun 2020 14:09:27 -0700 Subject: [PATCH 7/8] avoid child caching if filter cannot be transformed and lots of comment updates --- flow/layers/container_layer.h | 97 +++++++++++++------------------ flow/layers/image_filter_layer.cc | 25 ++++++-- flow/layers/image_filter_layer.h | 18 +++++- 3 files changed, 73 insertions(+), 67 deletions(-) diff --git a/flow/layers/container_layer.h b/flow/layers/container_layer.h index 8ce416e497ad9..1a276d32a2f36 100644 --- a/flow/layers/container_layer.h +++ b/flow/layers/container_layer.h @@ -56,35 +56,45 @@ class ContainerLayer : public Layer { }; //------------------------------------------------------------------------------ -/// Some ContainerLayer objects need to cache their children. The existing -/// layer caching mechanism only supports caching an individual layer as it -/// uses the unique_id() of that layer as the cache key. If a ContainerLayer -/// has multiple children and needs to cache them then it will need to first -/// collect them into a single layer object to satisfy the caching mechanism. -/// This utility class will silently collect all of the children of a -/// ContainerLayer into a single merged ContainerLayer automatically. +/// Some ContainerLayer objects perform a rendering operation or filter on +/// the rendered output of their children. Often that operation is changed +/// slightly from frame to frame as part of an animation. During such an +/// animation, the children can be cached if they are stable to avoid having +/// to render them on every frame. Even if the children are not stable, +/// rendering them into the raster cache during a Preroll operation will save +/// an extra change of rendering surface during the Paint phase as compared +/// to using the SaveLayer that would otherwise be needed with no caching. /// -/// The Flutter Widget objects that produce these layers tend to enforce -/// having only a single child Widget object which means that there is really -/// only a single child being added to this engine layer anyway, but due to -/// the complicated mechanism that turns trees of Widgets eventually into a -/// tree of engine layers, the associated layer may end up with multiple -/// direct child layers. This class implements a protection against the -/// incidental creation of "extra" children. +/// Typically the Flutter Widget objects that lead to the creation of these +/// layers will try to enforce only a single child Widget by their design. +/// Unfortunately, the process of turning Widgets eventually into engine +/// layers is not a 1:1 process so this layer might end up with multiple +/// child layers even if the Widget only had a single child Widget. /// -/// When the layer needs to recurse to perform some opertion on its children, +/// When such a layer goes to cache the output of its children, it will +/// need to supply a single layer to the cache mechanism since the raster +/// cache uses a layer unique_id() as part of the cache key. If this layer +/// ended up with multiple children, then it must first collect them into +/// one layer for the cache mechanism. In order to provide a single layer +/// for all of the children, this utility class will implicitly collect +/// the children into a secondary ContainerLayer called the child container. +/// +/// A by-product of creating a hidden child container, though, is that the +/// child container is created new every time this layer is created with +/// different properties, such as during an animation. In that scenario, +/// it would be best to cache the single real child of this layer if it +/// is unique and if it is stable from frame to frame. To facilitate this +/// optimal caching strategy, this class implements two accessor methods +/// to be used for different purposes: +/// +/// When the layer needs to recurse to perform some operation on its children, /// it can call GetChildContainer() to return the hidden container containing /// all of the real children. /// /// When the layer wants to cache the rendered contents of its children, it -/// should call GetCacheableChild() for best performance. -/// -/// Note that since the implicit child container is created new every time -/// this layer is recreated, the child container will be different for each -/// frame of an animation that modifies the properties of this layer. -/// The GetCacheableChild() method will attempt to find the single actual -/// child of this layer and return that instead since that child will more -/// likely remain stable across the frames of an animation. +/// should call GetCacheableChild() for best performance. This method may +/// end up returning the same layer as GetChildContainer(), but only if the +/// conditions for optimal caching of a single child are not met. /// class MergedContainerLayer : public ContainerLayer { public: @@ -94,19 +104,9 @@ class MergedContainerLayer : public ContainerLayer { protected: /** - * @brief Returns the ContainerLayer used to hold all of the children - * of the OpacityLayer. - * - * Often opacity layers will only have a single child since the associated - * Flutter widget is specified with only a single child widget pointer. - * But depending on the structure of the child tree that single widget at - * the framework level can turn into multiple children at the engine - * API level since there is no guarantee of a 1:1 correspondence of widgets - * to engine layers. This synthetic child container layer is established to - * hold all of the children in a single layer so that we can cache their - * output, but this synthetic layer will typically not be the best choice - * for the layer cache since the synthetic container is created fresh with - * each new OpacityLayer, and so may not be stable from frame to frame. + * @brief Returns the ContainerLayer used to hold all of the children of the + * MergedContainerLayer. Note that this may not be the best layer to use + * for caching the children. * * @see GetCacheableChild() * @return the ContainerLayer child used to hold the children @@ -115,30 +115,11 @@ class MergedContainerLayer : public ContainerLayer { /** * @brief Returns the best choice for a Layer object that can be used - * in RasterCache operations to cache the children of the OpacityLayer. + * in RasterCache operations to cache the children. * * The returned Layer must represent all children and try to remain stable - * if the OpacityLayer is reconstructed in subsequent frames of the scene. - * - * Note that since the synthetic child container returned from the - * GetChildContainer() method is created fresh with each new OpacityLayer, - * its return value will not be a good candidate for caching. But if the - * standard recommendations for animations are followed and the child widget - * is wrapped with a RepaintBoundary widget at the framework level, then - * the synthetic child container should contain the same single child layer - * on each frame. Under those conditions, that single child of the child - * container will be the best candidate for caching in the RasterCache - * and this method will return that single child if possible to improve - * the performance of caching the children. - * - * Note that if GetCacheableChild() does not find a single stable child of - * the child container it will return the child container as a fallback. - * Even though that child is new in each frame of an animation and thus we - * cannot reuse the cached layer raster between animation frames, the single - * container child will allow us to paint the child onto an offscreen buffer - * during Preroll() which reduces one render target switch compared to - * painting the child on the fly via an AutoSaveLayer in Paint() and thus - * still improves our performance. + * if the MergedContainerLayer is reconstructed in subsequent frames of + * the scene. * * @see GetChildContainer() * @return the best candidate Layer for caching the children diff --git a/flow/layers/image_filter_layer.cc b/flow/layers/image_filter_layer.cc index 448e81897aa8b..8b3e6fae9f096 100644 --- a/flow/layers/image_filter_layer.cc +++ b/flow/layers/image_filter_layer.cc @@ -7,7 +7,9 @@ namespace flutter { ImageFilterLayer::ImageFilterLayer(sk_sp filter) - : filter_(std::move(filter)), render_count_(1) {} + : filter_(std::move(filter)), + transformed_filter_(nullptr), + render_count_(1) {} void ImageFilterLayer::Preroll(PrerollContext* context, const SkMatrix& matrix) { @@ -27,10 +29,23 @@ void ImageFilterLayer::Preroll(PrerollContext* context, } set_paint_bounds(child_bounds); + transformed_filter_ = nullptr; if (render_count_ >= kMinimumRendersBeforeCachingFilterLayer) { + // We have rendered this same ImageFilterLayer object enough + // times to consider its properties and children to be stable + // from frame to frame so we try to cache the layer itself + // for maximum performance. TryToPrepareRasterCache(context, this, matrix); - } else { + } else if ((transformed_filter_ = filter_->makeWithLocalMatrix(matrix))) { + // This ImageFilterLayer is not yet considered stable so we + // increment the count to measure how many times it has been + // seen from frame to frame. render_count_++; + // And for now we will still try to cache the children as that + // provides a large performance benefit to avoid rendering the + // child layers themselves and also avoiding a rendering surface + // switch to do so during the Paint phase. This benefit is seen + // most during animations involving the ImageFilter. TryToPrepareRasterCache(context, GetCacheableChild(), matrix); } } @@ -43,11 +58,9 @@ void ImageFilterLayer::Paint(PaintContext& context) const { if (context.raster_cache->Draw(this, *context.leaf_nodes_canvas)) { return; } - const SkMatrix& ctm = context.leaf_nodes_canvas->getTotalMatrix(); - sk_sp transformed_filter = filter_->makeWithLocalMatrix(ctm); - if (transformed_filter) { + if (transformed_filter_) { SkPaint paint; - paint.setImageFilter(transformed_filter); + paint.setImageFilter(transformed_filter_); if (context.raster_cache->Draw(GetCacheableChild(), *context.leaf_nodes_canvas, &paint)) { diff --git a/flow/layers/image_filter_layer.h b/flow/layers/image_filter_layer.h index 02b003fd5bd8a..1df13bbd400d5 100644 --- a/flow/layers/image_filter_layer.h +++ b/flow/layers/image_filter_layer.h @@ -20,12 +20,24 @@ class ImageFilterLayer : public MergedContainerLayer { void Paint(PaintContext& context) const override; private: - // The default minimum number of times to render a filtered layer before - // we cache the output of the filter. Note that until this limit is - // reached we may continue to cache the children anyway. + // The ImageFilterLayer might cache the filtered output of this layer + // if the layer remains stable (if it is not animating for instance). + // If the ImageFilterLayer is not the same between rendered frames, + // though, it will cache its children instead and filter their cached + // output on the fly. + // Caching just the children saves the time to render them and also + // avoids a rendering surface switch to draw them. + // Caching the layer itself avoids all of that and additionally avoids + // the cost of applying the filter, but can be worse than caching the + // children if the filter itself is not stable from frame to frame. + // This constant controls how many times we will Preroll and Paint this + // same ImageFilterLayer before we consider the layer and filter to be + // stable enough to switch from caching the children to caching the + // filtered output of this layer. static constexpr int kMinimumRendersBeforeCachingFilterLayer = 3; sk_sp filter_; + sk_sp transformed_filter_; int render_count_; FML_DISALLOW_COPY_AND_ASSIGN(ImageFilterLayer); From 151a6d4951536fc543e3f542a10f540fd6b679d5 Mon Sep 17 00:00:00 2001 From: Jim Graham Date: Wed, 24 Jun 2020 18:05:12 -0700 Subject: [PATCH 8/8] adjust child caching logic per review feedback --- flow/layers/image_filter_layer.cc | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/flow/layers/image_filter_layer.cc b/flow/layers/image_filter_layer.cc index 8b3e6fae9f096..6f2d7aa59e407 100644 --- a/flow/layers/image_filter_layer.cc +++ b/flow/layers/image_filter_layer.cc @@ -36,17 +36,28 @@ void ImageFilterLayer::Preroll(PrerollContext* context, // from frame to frame so we try to cache the layer itself // for maximum performance. TryToPrepareRasterCache(context, this, matrix); - } else if ((transformed_filter_ = filter_->makeWithLocalMatrix(matrix))) { + } else { // This ImageFilterLayer is not yet considered stable so we // increment the count to measure how many times it has been // seen from frame to frame. render_count_++; - // And for now we will still try to cache the children as that - // provides a large performance benefit to avoid rendering the - // child layers themselves and also avoiding a rendering surface - // switch to do so during the Paint phase. This benefit is seen - // most during animations involving the ImageFilter. - TryToPrepareRasterCache(context, GetCacheableChild(), matrix); + + // Now we will try to pre-render the children into the cache. + // To apply the filter to pre-rendered children, we must first + // modify the filter to be aware of the transform under which + // the cached bitmap was produced. Some SkImageFilter + // instances can do this operation on some transforms and some + // (filters or transforms) cannot. We can only cache the children + // and apply the filter on the fly if this operation succeeds. + transformed_filter_ = filter_->makeWithLocalMatrix(matrix); + if (transformed_filter_) { + // With a modified SkImageFilter we can now try to cache the + // children to avoid their rendering costs if they remain + // stable between frames and also avoiding a rendering surface + // switch during the Paint phase even if they are not stable. + // This benefit is seen most during animations. + TryToPrepareRasterCache(context, GetCacheableChild(), matrix); + } } }