diff --git a/stdlib/public/core/Array.swift b/stdlib/public/core/Array.swift index 147b056665c48..bb7c72cd474a8 100644 --- a/stdlib/public/core/Array.swift +++ b/stdlib/public/core/Array.swift @@ -346,8 +346,7 @@ extension Array { @_semantics("array.make_mutable") internal mutating func _makeMutableAndUnique() { if _slowPath(!_buffer.isMutableAndUniquelyReferenced()) { - _createNewBuffer(bufferIsUnique: false, minimumCapacity: count, - growForAppend: false) + _buffer = _buffer._consumeAndCreateNew() } } @@ -1049,34 +1048,13 @@ extension Array: RangeReplaceableCollection { /// If `growForAppend` is true, the new capacity is calculated using /// `_growArrayCapacity`, but at least kept at `minimumCapacity`. @_alwaysEmitIntoClient - @inline(never) internal mutating func _createNewBuffer( bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool ) { - let newCapacity = _growArrayCapacity(oldCapacity: _getCapacity(), - minimumCapacity: minimumCapacity, - growForAppend: growForAppend) - let count = _getCount() - _internalInvariant(newCapacity >= count) - - let newBuffer = _ContiguousArrayBuffer( - _uninitializedCount: count, minimumCapacity: newCapacity) - - if bufferIsUnique { - _internalInvariant(_buffer.isUniquelyReferenced()) - - // As an optimization, if the original buffer is unique, we can just move - // the elements instead of copying. - let dest = newBuffer.firstElementAddress - dest.moveInitialize(from: _buffer.firstElementAddress, - count: count) - _buffer.count = 0 - } else { - _buffer._copyContents( - subRange: 0.. _ArrayBuffer { + return _consumeAndCreateNew(bufferIsUnique: false, + minimumCapacity: count, + growForAppend: false) + } + + /// Creates and returns a new uniquely referenced buffer which is a copy of + /// this buffer. + /// + /// If `bufferIsUnique` is true, the buffer is assumed to be uniquely + /// referenced and the elements are moved - instead of copied - to the new + /// buffer. + /// The `minimumCapacity` is the lower bound for the new capacity. + /// If `growForAppend` is true, the new capacity is calculated using + /// `_growArrayCapacity`, but at least kept at `minimumCapacity`. + /// + /// This buffer is consumed, i.e. it's released. + @_alwaysEmitIntoClient + @inline(never) + @_semantics("optimize.sil.specialize.owned2guarantee.never") + internal __consuming func _consumeAndCreateNew( + bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool + ) -> _ArrayBuffer { + let newCapacity = _growArrayCapacity(oldCapacity: capacity, + minimumCapacity: minimumCapacity, + growForAppend: growForAppend) + let c = count + _internalInvariant(newCapacity >= c) + + let newBuffer = _ContiguousArrayBuffer( + _uninitializedCount: c, minimumCapacity: newCapacity) + + if bufferIsUnique { + // As an optimization, if the original buffer is unique, we can just move + // the elements instead of copying. + let dest = newBuffer.firstElementAddress + dest.moveInitialize(from: firstElementAddress, + count: c) + _native.count = 0 + } else { + _copyContents( + subRange: 0..{ let eraseCount = subrange.count let growth = newCount - eraseCount - self.count = oldCount + growth + // This check will prevent storing a 0 count to the empty array singleton. + if growth != 0 { + self.count = oldCount + growth + } let elements = self.subscriptBaseAddress let oldTailIndex = subrange.upperBound diff --git a/stdlib/public/core/ContiguousArray.swift b/stdlib/public/core/ContiguousArray.swift index cc21a96f3661e..def207a400974 100644 --- a/stdlib/public/core/ContiguousArray.swift +++ b/stdlib/public/core/ContiguousArray.swift @@ -67,8 +67,7 @@ extension ContiguousArray { @_semantics("array.make_mutable") internal mutating func _makeMutableAndUnique() { if _slowPath(!_buffer.isMutableAndUniquelyReferenced()) { - _createNewBuffer(bufferIsUnique: false, minimumCapacity: count, - growForAppend: false) + _buffer = _buffer._consumeAndCreateNew() } } @@ -690,30 +689,10 @@ extension ContiguousArray: RangeReplaceableCollection { internal mutating func _createNewBuffer( bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool ) { - let newCapacity = _growArrayCapacity(oldCapacity: _getCapacity(), - minimumCapacity: minimumCapacity, - growForAppend: growForAppend) - let count = _getCount() - _internalInvariant(newCapacity >= count) - - let newBuffer = _ContiguousArrayBuffer( - _uninitializedCount: count, minimumCapacity: newCapacity) - - if bufferIsUnique { - _internalInvariant(_buffer.isUniquelyReferenced()) - - // As an optimization, if the original buffer is unique, we can just move - // the elements instead of copying. - let dest = newBuffer.firstElementAddress - dest.moveInitialize(from: _buffer.firstElementAddress, - count: count) - _buffer.count = 0 - } else { - _buffer._copyContents( - subRange: 0..: _ArrayBufferProtocol { return _isUnique(&_storage) } + /// Creates and returns a new uniquely referenced buffer which is a copy of + /// this buffer. + /// + /// This buffer is consumed, i.e. it's released. + @_alwaysEmitIntoClient + @inline(never) + @_semantics("optimize.sil.specialize.owned2guarantee.never") + internal __consuming func _consumeAndCreateNew() -> _ContiguousArrayBuffer { + return _consumeAndCreateNew(bufferIsUnique: false, + minimumCapacity: count, + growForAppend: false) + } + + /// Creates and returns a new uniquely referenced buffer which is a copy of + /// this buffer. + /// + /// If `bufferIsUnique` is true, the buffer is assumed to be uniquely + /// referenced and the elements are moved - instead of copied - to the new + /// buffer. + /// The `minimumCapacity` is the lower bound for the new capacity. + /// If `growForAppend` is true, the new capacity is calculated using + /// `_growArrayCapacity`, but at least kept at `minimumCapacity`. + /// + /// This buffer is consumed, i.e. it's released. + @_alwaysEmitIntoClient + @inline(never) + @_semantics("optimize.sil.specialize.owned2guarantee.never") + internal __consuming func _consumeAndCreateNew( + bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool + ) -> _ContiguousArrayBuffer { + let newCapacity = _growArrayCapacity(oldCapacity: capacity, + minimumCapacity: minimumCapacity, + growForAppend: growForAppend) + let c = count + _internalInvariant(newCapacity >= c) + + let newBuffer = _ContiguousArrayBuffer( + _uninitializedCount: c, minimumCapacity: newCapacity) + + if bufferIsUnique { + // As an optimization, if the original buffer is unique, we can just move + // the elements instead of copying. + let dest = newBuffer.firstElementAddress + dest.moveInitialize(from: firstElementAddress, + count: c) + count = 0 + } else { + _copyContents( + subRange: 0..(_ sequence: __owned S) where S.Element == Element { self.init() + // Needed to fully optimize OptionSet literals. + _onFastPath() for e in sequence { insert(e) } } diff --git a/test/IRGen/multithread_module.swift b/test/IRGen/multithread_module.swift index 895e710cc965c..ee82fac492696 100644 --- a/test/IRGen/multithread_module.swift +++ b/test/IRGen/multithread_module.swift @@ -68,8 +68,8 @@ public func mutateBaseArray(_ arr: inout [Base], _ x: Base) { // Check if all specializations from stdlib functions are created in the same LLVM module. -// CHECK-MAINLL-DAG: define {{.*}} @"$sSa16_createNewBuffer14bufferIsUnique15minimumCapacity13growForAppendySb_SiSbtF4test8MyStructV_Tg5" -// CHECK-MAINLL-DAG: define {{.*}} @"$sSa16_createNewBuffer14bufferIsUnique15minimumCapacity13growForAppendySb_SiSbtF4test4BaseC_Tg5" +// CHECK-MAINLL-DAG: define {{.*}} @"$ss{{(12_|22_Contiguous)}}ArrayBufferV20_consumeAndCreateNew14bufferIsUnique15minimumCapacity13growForAppendAByxGSb_SiSbtF4test8MyStructV_Tg5" +// CHECK-MAINLL-DAG: define {{.*}} @"$ss{{(12_|22_Contiguous)}}ArrayBufferV20_consumeAndCreateNew14bufferIsUnique15minimumCapacity13growForAppendAByxGSb_SiSbtF4test4BaseC_Tg5" // Check if the DI filename is correct and not "". diff --git a/test/SILOptimizer/array_contentof_opt.swift b/test/SILOptimizer/array_contentof_opt.swift index 7d2af225a2a79..1b1a3d5b9c649 100644 --- a/test/SILOptimizer/array_contentof_opt.swift +++ b/test/SILOptimizer/array_contentof_opt.swift @@ -1,4 +1,4 @@ -// RUN: %target-swift-frontend -O -sil-verify-all -emit-sil -Xllvm '-sil-inline-never-functions=$sSa6append' %s | %FileCheck %s +// RUN: %target-swift-frontend -O -sil-verify-all -emit-sil -Xllvm '-sil-inline-never-functions=$sSa6appendyy' %s | %FileCheck %s // REQUIRES: swift_stdlib_no_asserts,optimized_stdlib // This is an end-to-end test of the Array.append(contentsOf:) -> @@ -24,7 +24,7 @@ public func testInt(_ a: inout [Int]) { } // CHECK-LABEL: sil @{{.*}}testThreeInts -// CHECK-DAG: [[FR:%[0-9]+]] = function_ref @${{(sSa15reserveCapacityyySiFSi_Tg5|sSa16_createNewBuffer)}} +// CHECK-DAG: [[FR:%[0-9]+]] = function_ref @${{.*(reserveCapacity|_createNewBuffer)}} // CHECK-DAG: apply [[FR]] // CHECK-DAG: [[F:%[0-9]+]] = function_ref @$sSa6appendyyxnFSi_Tg5 // CHECK-DAG: apply [[F]] @@ -37,7 +37,7 @@ public func testThreeInts(_ a: inout [Int]) { // CHECK-LABEL: sil @{{.*}}testTooManyInts // CHECK-NOT: apply -// CHECK: [[F:%[0-9]+]] = function_ref @$sSa6append10contentsOfyqd__n_t7ElementQyd__RszSTRd__lFSi_SaySiGTg5 +// CHECK: [[F:%[0-9]+]] = function_ref @${{.*append.*contentsOf.*}} // CHECK-NOT: apply // CHECK: apply [[F]] // CHECK-NOT: apply @@ -65,12 +65,12 @@ public func dontPropagateContiguousArray(_ a: inout ContiguousArray) { // Check if the specialized Array.append(contentsOf:) is reasonably optimized for Array. -// CHECK-LABEL: sil shared {{.*}}@$sSa6append10contentsOfyqd__n_t7ElementQyd__RszSTRd__lFSi_SaySiGTg5Tf4gn_n +// CHECK-LABEL: sil shared {{.*}}@$sSa6append10contentsOfyqd__n_t7ElementQyd__RszSTRd__lFSi_SaySiGTg5 // There should only be a single call to _createNewBuffer or reserveCapacityForAppend/reserveCapacityImpl. // CHECK-NOT: apply -// CHECK: [[F:%[0-9]+]] = function_ref @{{.*(_createNewBuffer|reserveCapacity).*}} +// CHECK: [[F:%[0-9]+]] = function_ref @{{.*(_consumeAndCreateNew|reserveCapacity).*}} // CHECK-NEXT: apply [[F]] // CHECK-NOT: apply diff --git a/test/SILOptimizer/optionset.swift b/test/SILOptimizer/optionset.swift index 1ee43c2ef2135..6e332bc27281e 100644 --- a/test/SILOptimizer/optionset.swift +++ b/test/SILOptimizer/optionset.swift @@ -14,6 +14,7 @@ public struct TestOptions: OptionSet { // CHECK: sil @{{.*}}returnTestOptions{{.*}} // CHECK-NEXT: bb0: +// CHECK-NEXT: builtin // CHECK-NEXT: integer_literal {{.*}}, 15 // CHECK-NEXT: struct $Int // CHECK-NEXT: struct $TestOptions @@ -22,18 +23,19 @@ public func returnTestOptions() -> TestOptions { return [.first, .second, .third, .fourth] } -// CHECK: sil @{{.*}}returnEmptyTestOptions{{.*}} -// CHECK-NEXT: bb0: -// CHECK-NEXT: integer_literal {{.*}}, 0 -// CHECK-NEXT: struct $Int -// CHECK-NEXT: struct $TestOptions -// CHECK-NEXT: return +// CHECK: sil @{{.*}}returnEmptyTestOptions{{.*}} +// CHECK: [[ZERO:%[0-9]+]] = integer_literal {{.*}}, 0 +// CHECK: [[ZEROINT:%[0-9]+]] = struct $Int ([[ZERO]] +// CHECK: [[TO:%[0-9]+]] = struct $TestOptions ([[ZEROINT]] +// CHECK: return [[TO]] +// CHECK: } // end sil function {{.*}}returnEmptyTestOptions{{.*}} public func returnEmptyTestOptions() -> TestOptions { return [] } // CHECK: alloc_global @{{.*}}globalTestOptions{{.*}} // CHECK-NEXT: global_addr +// CHECK-NEXT: builtin // CHECK-NEXT: integer_literal {{.*}}, 15 // CHECK-NEXT: struct $Int // CHECK-NEXT: struct $TestOptions