diff --git a/README.md b/README.md index b3b43dd..ac3bbbe 100755 --- a/README.md +++ b/README.md @@ -557,6 +557,32 @@ The .NET 8 benchmarks include the number of cold and warm starts, alongside the 23.84 120.45 + + Minimal API on X86 + 195 + 1672.85 + 1737.61 + 1833.97 + 1889.15 + 136,006 + 6.30 + 10.98 + 26.72 + 124.67 + + + Minimal API on ARM64 + 242 + 1972.79 + 2049.16 + 2107.32 + 2124.55 + 136,816 + 6.01 + 9.37 + 24.69 + 331.6 + *The .NET 8 native AOT examples need to be compiled on Amazon Linux 2, this is a temporary solution as a pre-cursor to a SAM build image being available for .NET 8. diff --git a/loadtest/codebuild/load-test-buildspec.yml b/loadtest/codebuild/load-test-buildspec.yml index d2c1e0f..84bccfb 100644 --- a/loadtest/codebuild/load-test-buildspec.yml +++ b/loadtest/codebuild/load-test-buildspec.yml @@ -3,14 +3,23 @@ version: 0.2 phases: install: commands: - - npm i artillery -g + - cd loadtest/codebuild + - ./setup.sh build: commands: - - export temp1=$(pwd);echo $temp1 - - cd loadtest/codebuild - ./run-all-load-tests.sh artifacts: files: + - src/NET6/Report/* + - src/NET6Containers/Report/* + - src/NET6CustomRuntime/Report/* - src/NET6MinimalAPI/Report/* - src/NET6MinimalAPIWebAdapter/Report/* + - src/NET6Native/Report/* + - src/NET6TopLevelStatements/Report/* + - src/NET6WithPowerTools/Report/* + - src/NET8/Report/* + - src/NET8MinimalAPI/Report/* + - src/NET8Native/Report/* + - src/NET8NativeMinimalAPI/Report/* name: loadtest-$(date +%Y-%m-%H-%M) \ No newline at end of file diff --git a/loadtest/codebuild/load-test.yml b/loadtest/codebuild/load-test.yml new file mode 100755 index 0000000..75c7beb --- /dev/null +++ b/loadtest/codebuild/load-test.yml @@ -0,0 +1,32 @@ +config: + target: "{{ $processEnvironment.API_URL }}" + http: + timeout : 60 + processor: "../generator.js" + phases: + - duration: 60 + arrivalRate: 100 + +scenarios: + - name: "Generate products" + weight: 8 + flow: + - function: "generateProduct" + - put: + url: "/{{ Id }}" + headers: + Content-Type: "application/json" + json: + Id: "{{ Id }}" + Name: "{{ Name }}" + Price: "{{ Price }}" + - get: + url: "/{{ Id }}" + - think: 3 + - delete: + url: "/{{ Id }}" + - name: "Get products" + weight: 2 + flow: + - get: + url: "{{ $processEnvironment.API_URL }}" \ No newline at end of file diff --git a/loadtest/codebuild/run-all-load-tests.sh b/loadtest/codebuild/run-all-load-tests.sh index b2ac288..2febd69 100755 --- a/loadtest/codebuild/run-all-load-tests.sh +++ b/loadtest/codebuild/run-all-load-tests.sh @@ -32,7 +32,7 @@ echo -------------------------------------------- if [ "$LT_NET6_MINIMAL_API" != yes ]; then - echo SKIPPING net6 minimal api :$LT_NET6_MINIMAL_API + echo SKIPPING net6 minimal api - LT_NET6_MINIMAL_API=$LT_NET6_MINIMAL_API else echo "RUNNING load test for net6 minimal api" cd ../../src/NET6MinimalAPI/ @@ -42,10 +42,50 @@ fi if [ "$LT_NET6_MINIMAL_API_WEB_ADAPTER" != yes ]; then - echo SKIPPING net6 minimal api web adapter :$LT_NET6_MINIMAL_API_WEB_ADAPTER + echo SKIPPING net6 minimal api web adapter - LT_NET6_MINIMAL_API_WEB_ADAPTER = $LT_NET6_MINIMAL_API_WEB_ADAPTER else echo "RUNNING load test for net6 minimal api web adapter" cd ../../src/NET6MinimalAPIWebAdapter/ source ./deploy.sh $DELETE_STACK source ./run-loadtest.sh $TEST_DURATIOMN_SEC $LOG_INTERVAL_MIN $LOG_DELETE fi + +if [ "$LT_NET8" != yes ]; +then + echo SKIPPING net8 - LT_NET8=$LT_NET8 +else + echo "RUNNING load test for net8" + cd ../../src/NET8/ + source ./deploy.sh $DELETE_STACK + source ./run-loadtest.sh $TEST_DURATIOMN_SEC $LOG_INTERVAL_MIN $LOG_DELETE +fi + +if [ "$LT_NET8_MINIMAL_API" != yes ]; +then + echo SKIPPING net8 minimal api - LT_NET8_MINIMAL_API=$LT_NET8_MINIMAL_API +else + echo "RUNNING load test for net8 minimal api" + cd ../../src/NET8MinimalAPI/ + source ./deploy.sh $DELETE_STACK + source ./run-loadtest.sh $TEST_DURATIOMN_SEC $LOG_INTERVAL_MIN $LOG_DELETE +fi + +if [ "$LT_NET8_NATIVE" != yes ]; +then + echo SKIPPING net8 native - LT_NET8_NATIVE=$LT_NET8_NATIVE +else + echo "RUNNING load test for net8 native" + cd ../../src/NET8Native/ + source ./deploy.sh $DELETE_STACK + source ./run-loadtest.sh $TEST_DURATIOMN_SEC $LOG_INTERVAL_MIN $LOG_DELETE +fi + +if [ "$LT_NET8_NATIVE_MINIMAL_API" != yes ]; +then + echo SKIPPING net8 native minimal api LT_NET8_NATIVE_MINIMAL_API=$LT_NET8_NATIVE_MINIMAL_API +else + echo "RUNNING load test for net8 native minimal api" + cd ../../src/NET8NativeMinimalAPI/ + source ./deploy.sh $DELETE_STACK + source ./run-loadtest.sh $TEST_DURATIOMN_SEC $LOG_INTERVAL_MIN $LOG_DELETE +fi \ No newline at end of file diff --git a/loadtest/codebuild/setup.sh b/loadtest/codebuild/setup.sh new file mode 100755 index 0000000..ffa1a0b --- /dev/null +++ b/loadtest/codebuild/setup.sh @@ -0,0 +1,16 @@ +#LT_NET_SDK_CHANNEL +#Specifies the source channel for the installation. The possible values are: +# STS: The most recent Standard Term Support release. +# LTS: The most recent Long Term Support release. +# Two-part version in A.B format, representing a specific release (for example, 3.1 or 6.0). +# Three-part version in A.B.Cxx format, representing a specific SDK release (for example, 6.0.1xx or 6.0.2xx). Available since the 5.0 release. +echo -------------------------------------------- +echo Installing dotnet runtime version: $LT_NET_SDK_CHANNEL +echo -------------------------------------------- + +curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin --channel $LT_NET_SDK_CHANNEL + +echo -------------------------------------------- +echo Installing artillery +echo -------------------------------------------- +npm i artillery -g \ No newline at end of file diff --git a/src/NET6CustomRuntime/DeleteProduct/aws-lambda-tools-defaults.json b/src/NET6CustomRuntime/DeleteProduct/aws-lambda-tools-defaults.json deleted file mode 100644 index 1d112e0..0000000 --- a/src/NET6CustomRuntime/DeleteProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET6CustomRuntime/GetProduct/aws-lambda-tools-defaults.json b/src/NET6CustomRuntime/GetProduct/aws-lambda-tools-defaults.json deleted file mode 100644 index 1d112e0..0000000 --- a/src/NET6CustomRuntime/GetProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET6CustomRuntime/GetProducts/aws-lambda-tools-defaults.json b/src/NET6CustomRuntime/GetProducts/aws-lambda-tools-defaults.json deleted file mode 100644 index 1d112e0..0000000 --- a/src/NET6CustomRuntime/GetProducts/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET6CustomRuntime/PutProduct/aws-lambda-tools-defaults.json b/src/NET6CustomRuntime/PutProduct/aws-lambda-tools-defaults.json deleted file mode 100644 index 7ed1688..0000000 --- a/src/NET6CustomRuntime/PutProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Information": [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - "dotnet lambda help", - "All the command line options for the Lambda command can be specified in this file." - ], - "profile": "", - "region": "", - "configuration": "Release", - "function-runtime": "provided.al2", - "function-memory-size": 256, - "function-timeout": 30, - "function-handler": "bootstrap" -} \ No newline at end of file diff --git a/src/NET6MinimalAPI/Report/load-test-report-arm64.json b/src/NET6MinimalAPI/Report/load-test-report-arm64.json deleted file mode 100644 index 005bf4d..0000000 --- a/src/NET6MinimalAPI/Report/load-test-report-arm64.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "results": [ - [ - { - "field": "coldstart", - "value": "0" - }, - { - "field": "count", - "value": "135963" - }, - { - "field": "p50", - "value": "6.2055" - }, - { - "field": "p90", - "value": "9.6783" - }, - { - "field": "p99", - "value": "20.0868" - }, - { - "field": "max", - "value": "528.13" - } - ], - [ - { - "field": "coldstart", - "value": "1" - }, - { - "field": "count", - "value": "246" - }, - { - "field": "p50", - "value": "2105.2185" - }, - { - "field": "p90", - "value": "2164.9673" - }, - { - "field": "p99", - "value": "2215.3132" - }, - { - "field": "max", - "value": "2228.18" - } - ] - ], - "statistics": { - "recordsMatched": 136209.0, - "recordsScanned": 1226110.0, - "bytesScanned": 900312482.0 - }, - "status": "Complete" -} diff --git a/src/NET6MinimalAPI/Report/load-test-report-arm64.txt b/src/NET6MinimalAPI/Report/load-test-report-arm64.txt deleted file mode 100644 index 4ae278b..0000000 --- a/src/NET6MinimalAPI/Report/load-test-report-arm64.txt +++ /dev/null @@ -1,18 +0,0 @@ -Sun Nov 5 18:51:08 UTC 2023 -arm64 RESULTS lambda: Net6-MinimalApi-Arm64 -Test duration sec: 600 -Log interval min: 20 -Complete -RESULTS coldstart 0 -RESULTS count 135963 -RESULTS p50 6.2055 -RESULTS p90 9.6783 -RESULTS p99 20.0868 -RESULTS max 528.13 -RESULTS coldstart 1 -RESULTS count 246 -RESULTS p50 2105.2185 -RESULTS p90 2164.9673 -RESULTS p99 2215.3132 -RESULTS max 2228.18 -STATISTICS 900312482.0 136209.0 1226110.0 diff --git a/src/NET6MinimalAPI/Report/load-test-report-x86.json b/src/NET6MinimalAPI/Report/load-test-report-x86.json deleted file mode 100644 index d2d2920..0000000 --- a/src/NET6MinimalAPI/Report/load-test-report-x86.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "results": [ - [ - { - "field": "coldstart", - "value": "0" - }, - { - "field": "count", - "value": "135861" - }, - { - "field": "p50", - "value": "5.9169" - }, - { - "field": "p90", - "value": "9.9905" - }, - { - "field": "p99", - "value": "21.746" - }, - { - "field": "max", - "value": "108.6" - } - ], - [ - { - "field": "coldstart", - "value": "1" - }, - { - "field": "count", - "value": "197" - }, - { - "field": "p50", - "value": "1742.8361" - }, - { - "field": "p90", - "value": "1966.8893" - }, - { - "field": "p99", - "value": "2411.7468" - }, - { - "field": "max", - "value": "2503.31" - } - ] - ], - "statistics": { - "recordsMatched": 136058.0, - "recordsScanned": 1224739.0, - "bytesScanned": 894477163.0 - }, - "status": "Complete" -} diff --git a/src/NET6MinimalAPI/Report/load-test-report-x86.txt b/src/NET6MinimalAPI/Report/load-test-report-x86.txt deleted file mode 100644 index 21b5798..0000000 --- a/src/NET6MinimalAPI/Report/load-test-report-x86.txt +++ /dev/null @@ -1,18 +0,0 @@ -Sun Nov 5 18:40:27 UTC 2023 -x86 RESULTS lambda: Net6-MinimalApi-X86 -Test duration sec: 600 -Log interval min: 20 -Complete -RESULTS coldstart 0 -RESULTS count 135861 -RESULTS p50 5.9169 -RESULTS p90 9.9905 -RESULTS p99 21.746 -RESULTS max 108.6 -RESULTS coldstart 1 -RESULTS count 197 -RESULTS p50 1742.8361 -RESULTS p90 1966.8893 -RESULTS p99 2411.7468 -RESULTS max 2503.31 -STATISTICS 894477163.0 136058.0 1224739.0 diff --git a/src/NET6MinimalAPIWebAdapter/Report/load-test-report-arm64.json b/src/NET6MinimalAPIWebAdapter/Report/load-test-report-arm64.json deleted file mode 100644 index be65931..0000000 --- a/src/NET6MinimalAPIWebAdapter/Report/load-test-report-arm64.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "results": [ - [ - { - "field": "coldstart", - "value": "0" - }, - { - "field": "count", - "value": "138527" - }, - { - "field": "p50", - "value": "6.1078" - }, - { - "field": "p90", - "value": "9.3759" - }, - { - "field": "p99", - "value": "17.9744" - }, - { - "field": "max", - "value": "838.78" - } - ], - [ - { - "field": "coldstart", - "value": "1" - }, - { - "field": "count", - "value": "139" - }, - { - "field": "p50", - "value": "1277.1986" - }, - { - "field": "p90", - "value": "1326.6409" - }, - { - "field": "p99", - "value": "1358.8491" - }, - { - "field": "max", - "value": "1367.49" - } - ] - ], - "statistics": { - "recordsMatched": 138666.0, - "recordsScanned": 694392.0, - "bytesScanned": 268615241.0 - }, - "status": "Complete" -} diff --git a/src/NET6MinimalAPIWebAdapter/Report/load-test-report-arm64.txt b/src/NET6MinimalAPIWebAdapter/Report/load-test-report-arm64.txt deleted file mode 100644 index 6df5dbc..0000000 --- a/src/NET6MinimalAPIWebAdapter/Report/load-test-report-arm64.txt +++ /dev/null @@ -1,18 +0,0 @@ -Sun Nov 5 19:14:52 UTC 2023 -arm64 RESULTS lambda: Net6-MinimalApi-WebadApter-Arm64 -Test duration sec: 600 -Log interval min: 20 -Complete -RESULTS coldstart 0 -RESULTS count 138527 -RESULTS p50 6.1078 -RESULTS p90 9.3759 -RESULTS p99 17.9744 -RESULTS max 838.78 -RESULTS coldstart 1 -RESULTS count 139 -RESULTS p50 1277.1986 -RESULTS p90 1326.6409 -RESULTS p99 1358.8491 -RESULTS max 1367.49 -STATISTICS 268615241.0 138666.0 694392.0 diff --git a/src/NET6MinimalAPIWebAdapter/Report/load-test-report-x86.json b/src/NET6MinimalAPIWebAdapter/Report/load-test-report-x86.json deleted file mode 100644 index 60522ca..0000000 --- a/src/NET6MinimalAPIWebAdapter/Report/load-test-report-x86.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "results": [ - [ - { - "field": "coldstart", - "value": "0" - }, - { - "field": "count", - "value": "140872" - }, - { - "field": "p50", - "value": "6.2055" - }, - { - "field": "p90", - "value": "10.3128" - }, - { - "field": "p99", - "value": "21.746" - }, - { - "field": "max", - "value": "154.62" - } - ], - [ - { - "field": "coldstart", - "value": "1" - }, - { - "field": "count", - "value": "112" - }, - { - "field": "p50", - "value": "1013.88" - }, - { - "field": "p90", - "value": "1102.6789" - }, - { - "field": "p99", - "value": "1330.6248" - }, - { - "field": "max", - "value": "1392.85" - } - ] - ], - "statistics": { - "recordsMatched": 140984.0, - "recordsScanned": 705687.0, - "bytesScanned": 272986184.0 - }, - "status": "Complete" -} diff --git a/src/NET6MinimalAPIWebAdapter/Report/load-test-report-x86.txt b/src/NET6MinimalAPIWebAdapter/Report/load-test-report-x86.txt deleted file mode 100644 index 1f5f629..0000000 --- a/src/NET6MinimalAPIWebAdapter/Report/load-test-report-x86.txt +++ /dev/null @@ -1,18 +0,0 @@ -Sun Nov 5 19:04:10 UTC 2023 -x86 RESULTS lambda: Net6-MinimalApi-WebadApter-X86 -Test duration sec: 600 -Log interval min: 20 -Complete -RESULTS coldstart 0 -RESULTS count 140872 -RESULTS p50 6.2055 -RESULTS p90 10.3128 -RESULTS p99 21.746 -RESULTS max 154.62 -RESULTS coldstart 1 -RESULTS count 112 -RESULTS p50 1013.88 -RESULTS p90 1102.6789 -RESULTS p99 1330.6248 -RESULTS max 1392.85 -STATISTICS 272986184.0 140984.0 705687.0 diff --git a/src/NET6Native/DeleteProduct/aws-lambda-tools-defaults.json b/src/NET6Native/DeleteProduct/aws-lambda-tools-defaults.json deleted file mode 100755 index 1d112e0..0000000 --- a/src/NET6Native/DeleteProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET6Native/GetProduct/aws-lambda-tools-defaults.json b/src/NET6Native/GetProduct/aws-lambda-tools-defaults.json deleted file mode 100755 index 1d112e0..0000000 --- a/src/NET6Native/GetProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET6Native/GetProducts/aws-lambda-tools-defaults.json b/src/NET6Native/GetProducts/aws-lambda-tools-defaults.json deleted file mode 100755 index 1d112e0..0000000 --- a/src/NET6Native/GetProducts/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET6Native/PutProduct/aws-lambda-tools-defaults.json b/src/NET6Native/PutProduct/aws-lambda-tools-defaults.json deleted file mode 100755 index 1d112e0..0000000 --- a/src/NET6Native/PutProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET8/DeleteProduct/DeleteProduct.csproj b/src/NET8/DeleteProduct/DeleteProduct.csproj index 825e086..a386b03 100644 --- a/src/NET8/DeleteProduct/DeleteProduct.csproj +++ b/src/NET8/DeleteProduct/DeleteProduct.csproj @@ -15,12 +15,12 @@ - - - - - - + + + + + + diff --git a/src/NET8/DeleteProduct/aws-lambda-tools-defaults.json b/src/NET8/DeleteProduct/aws-lambda-tools-defaults.json deleted file mode 100644 index 1d112e0..0000000 --- a/src/NET8/DeleteProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET8/GenerateLoadTestResults/DateUtils.cs b/src/NET8/GenerateLoadTestResults/DateUtils.cs deleted file mode 100644 index c37c43d..0000000 --- a/src/NET8/GenerateLoadTestResults/DateUtils.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; - -namespace GenerateLoadTestResults; - -public static class DateUtils -{ - public static long AsUnixTimestamp(this DateTime date) - { - DateTime origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); - TimeSpan diff = date.ToUniversalTime() - origin; - return (long)Math.Floor(diff.TotalSeconds); - } -} \ No newline at end of file diff --git a/src/NET8/GenerateLoadTestResults/Function.cs b/src/NET8/GenerateLoadTestResults/Function.cs deleted file mode 100644 index 21a57a8..0000000 --- a/src/NET8/GenerateLoadTestResults/Function.cs +++ /dev/null @@ -1,146 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text.Json; -using System.Threading.Tasks; -using Amazon.CloudWatchLogs; -using Amazon.CloudWatchLogs.Model; -using Amazon.Lambda.APIGatewayEvents; -using Amazon.Lambda.Core; - -[assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.SystemTextJson.DefaultLambdaJsonSerializer))] - -namespace GenerateLoadTestResults -{ - public class Function - { - private AmazonCloudWatchLogsClient _cloudWatchLogsClient; - - public Function() - { - this._cloudWatchLogsClient = new AmazonCloudWatchLogsClient(); - } - - public async Task FunctionHandler( - APIGatewayHttpApiV2ProxyRequest apigProxyEvent, - ILambdaContext context) - { - if (!apigProxyEvent.RequestContext.Http.Method.Equals(HttpMethod.Get.Method)) - { - return new APIGatewayHttpApiV2ProxyResponse - { - Body = "Only GET allowed", - StatusCode = (int) HttpStatusCode.MethodNotAllowed, - }; - } - - try - { - var resultRows = 0; - var queryCount = 0; - - List> finalResults = new List>(); - - while (resultRows < 2 || queryCount >= 3) - { - finalResults = await runQuery(context); - - resultRows = finalResults.Count; - queryCount++; - } - - var wrapper = new QueryResultWrapper() - { - LoadTestType = - $"{Environment.GetEnvironmentVariable("LOAD_TEST_TYPE")} ({Environment.GetEnvironmentVariable("LAMBDA_ARCHITECTURE")})", - WarmStart = new QueryResult() - { - Count = finalResults[0][1].Value, - P50 = finalResults[0][2].Value, - P90 = finalResults[0][3].Value, - P99 = finalResults[0][4].Value, - Max = finalResults[0][5].Value, - }, - ColdStart = new QueryResult() - { - Count = finalResults[1][1].Value, - P50 = finalResults[1][2].Value, - P90 = finalResults[1][3].Value, - P99 = finalResults[1][4].Value, - Max = finalResults[1][5].Value, - } - }; - - return new APIGatewayHttpApiV2ProxyResponse - { - StatusCode = (int) HttpStatusCode.OK, - Body = wrapper.AsMarkdownTableRow(), - Headers = new Dictionary {{"Content-Type", "text/html"}} - }; - } - catch (Exception e) - { - context.Logger.LogLine($"Error retrieving results {e.Message} {e.StackTrace}"); - - return new APIGatewayHttpApiV2ProxyResponse - { - Body = "Not Found", - StatusCode = (int) HttpStatusCode.InternalServerError, - }; - } - } - - private async Task>> runQuery(ILambdaContext context) - { - var logGroupNamePrefix = - $"{Environment.GetEnvironmentVariable("LOG_GROUP_PREFIX")}{Environment.GetEnvironmentVariable("LAMBDA_ARCHITECTURE")}" - .Replace("_", "-"); - - context.Logger.LogLine($"Retrieving log groups with prefix {logGroupNamePrefix}"); - - var logGroupList = await _cloudWatchLogsClient.DescribeLogGroupsAsync(new DescribeLogGroupsRequest() - { - LogGroupNamePrefix = logGroupNamePrefix, - }); - - context.Logger.LogLine($"Found {logGroupList.LogGroups.Count} log group(s)"); - - var queryRes = await _cloudWatchLogsClient.StartQueryAsync(new StartQueryRequest() - { - LogGroupNames = logGroupList.LogGroups.Select(p => p.LogGroupName).ToList(), - QueryString = - "filter @type=\"REPORT\" | fields greatest(@initDuration, 0) + @duration as duration, ispresent(@initDuration) as coldstart | stats count(*) as count, pct(duration, 50) as p50, pct(duration, 90) as p90, pct(duration, 99) as p99, max(duration) as max by coldstart", - StartTime = DateTime.Now.AddMinutes(-20).AsUnixTimestamp(), - EndTime = DateTime.Now.AsUnixTimestamp(), - }); - - context.Logger.LogLine($"Running query, query id is {queryRes.QueryId}"); - - QueryStatus currentQueryStatus = QueryStatus.Running; - List> finalResults = new List>(); - - while (currentQueryStatus == QueryStatus.Running || currentQueryStatus == QueryStatus.Scheduled) - { - context.Logger.LogLine("Retrieving query results"); - - var queryResults = await _cloudWatchLogsClient.GetQueryResultsAsync(new GetQueryResultsRequest() - { - QueryId = queryRes.QueryId - }); - - context.Logger.LogLine($"Query result status is {queryResults.Status}"); - - currentQueryStatus = queryResults.Status; - finalResults = queryResults.Results; - - await Task.Delay(TimeSpan.FromSeconds(5)); - } - - context.Logger.LogLine($"Final results: {finalResults.Count} row(s)"); - - return finalResults; - } - } -} \ No newline at end of file diff --git a/src/NET8/GenerateLoadTestResults/GenerateLoadTestResults.csproj b/src/NET8/GenerateLoadTestResults/GenerateLoadTestResults.csproj deleted file mode 100644 index 4a8d708..0000000 --- a/src/NET8/GenerateLoadTestResults/GenerateLoadTestResults.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - net6.0 - true - true - - - - - - - - - - - diff --git a/src/NET8/GenerateLoadTestResults/QueryResult.cs b/src/NET8/GenerateLoadTestResults/QueryResult.cs deleted file mode 100644 index 69b5a31..0000000 --- a/src/NET8/GenerateLoadTestResults/QueryResult.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace GenerateLoadTestResults; - -public record QueryResultWrapper -{ - public string LoadTestType { get; set; } - - public QueryResult ColdStart { get; set; } - - public QueryResult WarmStart { get; set; } - - public string AsMarkdownTableRow() => $"
Cold Start (ms)Warm Start (ms)
p50p90p99maxp50p90p99max
{LoadTestType}{ColdStart.P50}{ColdStart.P90}{ColdStart.P99}{ColdStart.Max}{WarmStart.P50}{WarmStart.P90}{WarmStart.P99}{WarmStart.Max}
"; -} - -public record QueryResult -{ - public string Count { get; set; } - - public string P50 { get; set; } - - public string P90 { get; set; } - - public string P99 { get; set; } - - public string Max { get; set; } -} \ No newline at end of file diff --git a/src/NET8/GetProduct/GetProduct.csproj b/src/NET8/GetProduct/GetProduct.csproj index 9757d2d..8253d9e 100644 --- a/src/NET8/GetProduct/GetProduct.csproj +++ b/src/NET8/GetProduct/GetProduct.csproj @@ -13,12 +13,12 @@ linux-x64 - - - - - - + + + + + + diff --git a/src/NET8/GetProduct/aws-lambda-tools-defaults.json b/src/NET8/GetProduct/aws-lambda-tools-defaults.json deleted file mode 100644 index 1d112e0..0000000 --- a/src/NET8/GetProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET8/GetProducts/GetProducts.csproj b/src/NET8/GetProducts/GetProducts.csproj index 55460a1..982b1ac 100644 --- a/src/NET8/GetProducts/GetProducts.csproj +++ b/src/NET8/GetProducts/GetProducts.csproj @@ -14,12 +14,12 @@ linux-x64 - - - - - - + + + + + + diff --git a/src/NET8/GetProducts/aws-lambda-tools-defaults.json b/src/NET8/GetProducts/aws-lambda-tools-defaults.json deleted file mode 100644 index 1d112e0..0000000 --- a/src/NET8/GetProducts/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET8/Makefile b/src/NET8/Makefile index 2aadd31..4850513 100644 --- a/src/NET8/Makefile +++ b/src/NET8/Makefile @@ -1,15 +1,23 @@ -build-GetProductFunction: - dotnet clean +build-GetProductFunctionX86: dotnet publish GetProduct/GetProduct.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) -build-GetProductsFunction: - dotnet clean +build-GetProductsFunctionX86: dotnet publish GetProducts/GetProducts.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) -build-PutProductFunction: - dotnet clean +build-PutProductFunctionX86: dotnet publish PutProduct/PutProduct.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) -build-DeleteProductFunction: - dotnet clean - dotnet publish DeleteProduct/DeleteProduct.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) \ No newline at end of file +build-DeleteProductFunctionX86: + dotnet publish DeleteProduct/DeleteProduct.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) + +build-GetProductFunctionArm64: + dotnet publish GetProduct/GetProduct.csproj -c Release -r linux-arm64 --self-contained -o $(ARTIFACTS_DIR) + +build-GetProductsFunctionArm64: + dotnet publish GetProducts/GetProducts.csproj -c Release -r linux-arm64 --self-contained -o $(ARTIFACTS_DIR) + +build-PutProductFunctionArm64: + dotnet publish PutProduct/PutProduct.csproj -c Release -r linux-arm64 --self-contained -o $(ARTIFACTS_DIR) + +build-DeleteProductFunctionArm64: + dotnet publish DeleteProduct/DeleteProduct.csproj -c Release -r linux-arm64 --self-contained -o $(ARTIFACTS_DIR) \ No newline at end of file diff --git a/src/NET8/PutProduct/PutProduct.csproj b/src/NET8/PutProduct/PutProduct.csproj index 9fae9e3..3f79dae 100644 --- a/src/NET8/PutProduct/PutProduct.csproj +++ b/src/NET8/PutProduct/PutProduct.csproj @@ -15,12 +15,12 @@ - - - - - - + + + + + + diff --git a/src/NET8/PutProduct/aws-lambda-tools-defaults.json b/src/NET8/PutProduct/aws-lambda-tools-defaults.json deleted file mode 100644 index 7ed1688..0000000 --- a/src/NET8/PutProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Information": [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - "dotnet lambda help", - "All the command line options for the Lambda command can be specified in this file." - ], - "profile": "", - "region": "", - "configuration": "Release", - "function-runtime": "provided.al2", - "function-memory-size": 256, - "function-timeout": 30, - "function-handler": "bootstrap" -} \ No newline at end of file diff --git a/src/NET8/Shared/Shared.csproj b/src/NET8/Shared/Shared.csproj index 31e5c9f..2797db9 100644 --- a/src/NET8/Shared/Shared.csproj +++ b/src/NET8/Shared/Shared.csproj @@ -13,10 +13,10 @@ linux-x64 - - - - + + + + diff --git a/src/NET8/deploy.sh b/src/NET8/deploy.sh new file mode 100755 index 0000000..814ecb5 --- /dev/null +++ b/src/NET8/deploy.sh @@ -0,0 +1,43 @@ +#Arguments: +#$1 - delete stack formation to ensure that all test have same conditon and favour similar ammount of cold start events + +STACK_NAME=dotnet8 +DELETE_STACK=yes + +COLOR='\033[0;33m' +NO_COLOR='\033[0m' # No Color + +if [ "x$1" != x ]; +then + DELETE_STACK=$1 +fi + +echo "${COLOR}" +echo -------------------------------------------- +echo DELETE_STACK: $DELETE_STACK +echo -------------------------------------------- +echo "${NO_COLOR}" + +if [ $DELETE_STACK == "yes" ]; +then + echo "${COLOR}" + echo -------------------------------------------- + echo DELETING STACK $STACK_NAME + echo -------------------------------------------- + echo "${NO_COLOR}" + aws cloudformation delete-stack --stack-name $STACK_NAME + echo "${COLOR}" + echo --------------------------------------------- + echo Waiting stack to be deleted + echo -------------------------------------------- + echo "${NO_COLOR}" + aws cloudformation wait stack-delete-complete --stack-name $STACK_NAME + echo "${COLOR}" + echo --------------------------------------------- + echo Stack deleted + echo -------------------------------------------- + echo "${NO_COLOR}" +fi + +sam build +sam deploy --stack-name $STACK_NAME --resolve-s3 --s3-prefix $STACK_NAME --no-confirm-changeset --no-fail-on-empty-changeset --capabilities CAPABILITY_IAM \ No newline at end of file diff --git a/src/NET8/run-loadtest.sh b/src/NET8/run-loadtest.sh new file mode 100755 index 0000000..58d219a --- /dev/null +++ b/src/NET8/run-loadtest.sh @@ -0,0 +1,182 @@ +#Arguments: +#$1 - load test duration in seconds +#$2 - log interval to be used in the cloudwatch query in minutes +#$3 - when equal to yes cloudwatch log group will be deleted to ensure that only logs of the load test will be evaluated for stat +#$4 - ARN of sns topic to notify test results + +STACK_NAME=dotnet8 +TEST_DURATIOMN_SEC=60 +LOG_INTERVAL_MIN=20 +LOG_DELETE=yes + +COLOR='\033[0;33m' +NO_COLOR='\033[0m' # No Color + +if [ "x$1" != x ]; +then + TEST_DURATIOMN_SEC=$1 +fi + +if [ "x$2" != x ]; +then + LOG_INTERVAL_MIN=$2 +fi + +if [ "x$3" != x ]; +then + LOG_DELETE=$3 +fi + +if [ "x$4" != x ]; +then + SNS_TOPIC_ARN=$4 +fi + +echo "${COLOR}" +echo -------------------------------------------- +echo DURATION:$TEST_DURATIOMN_SEC +echo LOG INTERVAL:$LOG_INTERVAL_MIN +echo LOG_DELETE: $LOG_DELETE +echo SNS_TOPIC_ARN: $SNS_TOPIC_ARN +echo -------------------------------------------- +echo "${NO_COLOR}" + +mkdir -p Report + +function RunLoadTest() +{ + #Params: + #$1 - Architecture (x86 or arm64).Used for logging and naming report file + #$2 - Stack output name to get API Url + #$3 - Stack output name to get lambda name GetProducts + #$4 - Stack output name to get lambda name GetProduct + #$5 - Stack output name to get lambda name DeleteProduct + #$6 - Stack output name to get lambda name PutProduct + + #get test params from cloud formation output + echo "${COLOR}" + export API_URL=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$2'].OutputValue" \ + --output text) + echo API URL: $API_URL + + LAMBDA_GETPRODUCTS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$3'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA_GETPRODUCTS + + LAMBDA_GETPRODUCT=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$4'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA_GETPRODUCT + + LAMBDA_DELETEPRODUCT=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$5'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA_DELETEPRODUCT + + LAMBDA_PUTPRODUCT=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$6'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA_PUTPRODUCT + + if [ $LOG_DELETE == "yes" ]; + then + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA_GETPRODUCTS + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA_GETPRODUCTS + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA_GETPRODUCT + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA_GETPRODUCT + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA_DELETEPRODUCT + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA_DELETEPRODUCT + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA_PUTPRODUCT + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA_PUTPRODUCT + echo --------------------------------------------- + echo Waiting 10 sec. for deletion to complete + echo -------------------------------------------- + sleep 10 + fi + + #run load test with artillery + echo -------------------------------------------- + echo $1 RUNNING LOAD TEST $TEST_DURATIOMN_SEC sec $API_URL + echo -------------------------------------------- + echo "${NO_COLOR}" + artillery run \ + --overrides '{"config": { "phases": [{ "duration": '$TEST_DURATIOMN_SEC', "arrivalRate": 100 }] } }' \ + --quiet \ + ../../loadtest/codebuild/load-test.yml + + echo "${COLOR}" + echo -------------------------------------------- + echo Waiting 10 sec. for logs to consolidate + echo -------------------------------------------- + sleep 10 + + #get stats from cloudwatch + enddate=$(date "+%s") + startdate=$(($enddate-($LOG_INTERVAL_MIN*60))) + echo -------------------------------------------- + echo Log start:$startdate end:$enddate + echo -------------------------------------------- + + echo -------------------------------------------- + echo GET ERROR METRICS $1 + echo -------------------------------------------- + aws cloudwatch get-metric-statistics \ + --namespace AWS/Lambda \ + --metric-name Errors \ + --dimensions Name=FunctionName,Value=$LAMBDA_GETPRODUCTS Name=FunctionName,Value=$LAMBDA_GETPRODUCT Name=FunctionName,Value=$LAMBDA_DELETEPRODUCT Name=FunctionName,Value=$LAMBDA_PUTPRODUCT \ + --statistics Sum --period 43200 \ + --start-time $startdate --end-time $enddate > ./Report/load-test-errors-$1.json + + cat ./Report/load-test-errors-$1.json + + QUERY_ID=$(aws logs start-query \ + --log-group-names "/aws/lambda/$LAMBDA_GETPRODUCTS" "/aws/lambda/$LAMBDA_GETPRODUCT" "/aws/lambda/$LAMBDA_DELETEPRODUCT" "/aws/lambda/$LAMBDA_PUTPRODUCT" \ + --start-time $startdate \ + --end-time $enddate \ + --query-string 'filter @type="REPORT" | fields greatest(@initDuration, 0) + @duration as duration, ispresent(@initDuration) as coldstart | stats count(*) as count, pct(duration, 50) as p50, pct(duration, 90) as p90, pct(duration, 99) as p99, max(duration) as max by coldstart' \ + | jq -r '.queryId') + + echo -------------------------------------------- + echo Query started, id: $QUERY_ID + echo -------------------------------------------- + + echo --------------------------------------------- + echo Waiting 10 sec. for cloudwatch query to complete + echo -------------------------------------------- + sleep 10 + + echo -------------------------------------------- + echo RESULTS $1 + echo -------------------------------------------- + echo "${NO_COLOR}" + date > ./Report/load-test-report-$1.txt + echo $1 RESULTS >> ./Report/load-test-report-$1.txt + echo Test duration sec: $TEST_DURATIOMN_SEC >> ./Report/load-test-report-$1.txt + echo Log interval min: $LOG_INTERVAL_MIN >> ./Report/load-test-report-$1.txt + aws logs get-query-results --query-id $QUERY_ID --output text >> ./Report/load-test-report-$1.txt + cat ./Report/load-test-report-$1.txt + aws logs get-query-results --query-id $QUERY_ID --output json >> ./Report/load-test-report-$1.json + + if [ "x$SNS_TOPIC_ARN" != x ]; + then + echo -------------------------------------------- + echo Sending message to sns topic: $SNS_TOPIC_ARN + echo -------------------------------------------- + msg=$(<./Report/load-test-report-$1.txt)\n\n$(<./Report/load-test-errors-$1.json) + subject="serverless dotnet demo load test result for $LAMBDA_GETPRODUCTS" + aws sns publish --topic-arn $SNS_TOPIC_ARN --subject "$subject" --message "$msg" + fi +} + +RunLoadTest x86 ApiUrlX86 LambdaX86NameGetProducts LambdaX86NameGetProduct LambdaX86NameDeleteProduct LambdaX86NamePutProduct +RunLoadTest arm64 ApiUrlArm64 LambdaArm64NameGetProducts LambdaArm64NameGetProduct LambdaArm64NameDeleteProduct LambdaArm64NamePutProduct \ No newline at end of file diff --git a/src/NET8/template.yaml b/src/NET8/template.yaml index c6c34f7..0e49fe0 100644 --- a/src/NET8/template.yaml +++ b/src/NET8/template.yaml @@ -1,10 +1,17 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 +Parameters: + x86FunctionNamePrefix: + Type: String + Default: Net8-X86 + arm64FunctionNamePrefix: + Type: String + Default: Net8-Arm64 + Globals: Function: MemorySize: 1024 - Architectures: ["x86_64"] Runtime: provided.al2 Timeout: 30 Tracing: Active @@ -13,16 +20,19 @@ Globals: PRODUCT_TABLE_NAME: !Ref Table Resources: - GetProductsFunction: + #X86 + GetProductsFunctionX86: Type: AWS::Serverless::Function Properties: + FunctionName: !Join [ -,[!Ref x86FunctionNamePrefix,GetProducts]] + Architectures: [x86_64] CodeUri: ./ Handler: bootstrap Events: Api: Type: HttpApi Properties: - Path: / + Path: /x86 Method: GET Policies: - DynamoDBReadPolicy: @@ -31,16 +41,18 @@ Resources: Metadata: BuildMethod: makefile - GetProductFunction: + GetProductFunctionX86: Type: AWS::Serverless::Function Properties: + FunctionName: !Join [ -,[!Ref x86FunctionNamePrefix,GetProduct]] + Architectures: [x86_64] CodeUri: ./ Handler: GetProduct Events: Api: Type: HttpApi Properties: - Path: /{id} + Path: /x86/{id} Method: GET Policies: - Version: "2012-10-17" @@ -51,16 +63,18 @@ Resources: Metadata: BuildMethod: makefile - DeleteProductFunction: + DeleteProductFunctionX86: Type: AWS::Serverless::Function Properties: + FunctionName: !Join [ -,[!Ref x86FunctionNamePrefix,DeleteProduct]] + Architectures: [x86_64] CodeUri: ./ Handler: DeleteProduct Events: Api: Type: HttpApi Properties: - Path: /{id} + Path: /x86/{id} Method: DELETE Policies: - Version: "2012-10-17" @@ -73,16 +87,18 @@ Resources: Metadata: BuildMethod: makefile - PutProductFunction: + PutProductFunctionX86: Type: AWS::Serverless::Function Properties: + FunctionName: !Join [ -,[!Ref x86FunctionNamePrefix,PutProduct]] + Architectures: [x86_64] CodeUri: ./ Handler: PutProduct Events: Api: Type: HttpApi Properties: - Path: /{id} + Path: /x86/{id} Method: PUT Policies: - Version: "2012-10-17" @@ -92,37 +108,94 @@ Resources: Resource: !GetAtt Table.Arn Metadata: BuildMethod: makefile + #ARM64 + GetProductsFunctionArm64: + Type: AWS::Serverless::Function + Properties: + FunctionName: !Join [ -,[!Ref arm64FunctionNamePrefix,GetProducts]] + Architectures: [arm64] + CodeUri: ./ + Handler: bootstrap + Events: + Api: + Type: HttpApi + Properties: + Path: /arm64 + Method: GET + Policies: + - DynamoDBReadPolicy: + TableName: + !Ref Table + Metadata: + BuildMethod: makefile - GenerateLoadTestResults: + GetProductFunctionArm64: Type: AWS::Serverless::Function Properties: - CodeUri: ./GenerateLoadTestResults/ - Handler: GenerateLoadTestResults::GenerateLoadTestResults.Function::FunctionHandler - Runtime: dotnet6 + FunctionName: !Join [ -,[!Ref arm64FunctionNamePrefix,GetProduct]] + Architectures: [arm64] + CodeUri: ./ + Handler: GetProduct Events: Api: Type: HttpApi Properties: - Path: /test-results + Path: /arm64/{id} Method: GET - Environment: - Variables: - LOG_GROUP_PREFIX: !Sub "/aws/lambda/net-8-base-" - LOAD_TEST_TYPE: "NET 8" - LAMBDA_ARCHITECTURE: "x86_64" Policies: - Version: "2012-10-17" Statement: - - Sid: AllowStartQueries - Effect: Allow + - Effect: Allow + Action: dynamodb:GetItem + Resource: !GetAtt Table.Arn + Metadata: + BuildMethod: makefile + + DeleteProductFunctionArm64: + Type: AWS::Serverless::Function + Properties: + FunctionName: !Join [ -,[!Ref arm64FunctionNamePrefix,DeleteProduct]] + Architectures: [arm64] + CodeUri: ./ + Handler: DeleteProduct + Events: + Api: + Type: HttpApi + Properties: + Path: /arm64/{id} + Method: DELETE + Policies: + - Version: "2012-10-17" + Statement: + - Effect: Allow Action: - - logs:DescribeLogGroups - - logs:StartQuery - Resource: "*" - - Sid: AllowGetQueryResults - Effect: Allow - Action: logs:GetQueryResults - Resource: "*" + - dynamodb:DeleteItem + - dynamodb:GetItem + Resource: !GetAtt Table.Arn + Metadata: + BuildMethod: makefile + + PutProductFunctionArm64: + Type: AWS::Serverless::Function + Properties: + FunctionName: !Join [ -,[!Ref arm64FunctionNamePrefix,PutProduct]] + Architectures: [arm64] + CodeUri: ./ + Handler: PutProduct + Events: + Api: + Type: HttpApi + Properties: + Path: /arm64/{id} + Method: PUT + Policies: + - Version: "2012-10-17" + Statement: + - Effect: Allow + Action: dynamodb:PutItem + Resource: !GetAtt Table.Arn + Metadata: + BuildMethod: makefile Table: Type: AWS::DynamoDB::Table @@ -138,4 +211,38 @@ Resources: Outputs: ApiUrl: Description: "API Gateway endpoint URL" - Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/" \ No newline at end of file + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com" + ApiUrlX86: + Description: "X86 API endpoint URL" + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/x86" + ApiUrlArm64: + Description: "Arm64 GateAPI endpoint URL" + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/arm64" + + #x86 + LambdaX86NameGetProducts: + Description: "Lambda X86 GetProducts" + Value: !Join [ -,[!Ref x86FunctionNamePrefix,GetProducts]] + LambdaX86NameGetProduct: + Description: "Lambda X86 GetProduct" + Value: !Join [ -,[!Ref x86FunctionNamePrefix,GetProduct]] + LambdaX86NameDeleteProduct: + Description: "Lambda X86 DeleteProduct" + Value: !Join [ -,[!Ref x86FunctionNamePrefix,DeleteProduct]] + LambdaX86NamePutProduct: + Description: "Lambda X86 PutProduct" + Value: !Join [ -,[!Ref x86FunctionNamePrefix,PutProduct]] + + #arm64 + LambdaArm64NameGetProducts: + Description: "Lambda X86 GetProducts" + Value: !Join [ -,[!Ref arm64FunctionNamePrefix,GetProducts]] + LambdaArm64NameGetProduct: + Description: "Lambda X86 GetProduct" + Value: !Join [ -,[!Ref arm64FunctionNamePrefix,GetProduct]] + LambdaArm64NameDeleteProduct: + Description: "Lambda X86 DeleteProduct" + Value: !Join [ -,[!Ref arm64FunctionNamePrefix,DeleteProduct]] + LambdaArm64NamePutProduct: + Description: "Lambda X86 PutProduct" + Value: !Join [ -,[!Ref arm64FunctionNamePrefix,PutProduct]] \ No newline at end of file diff --git a/src/NET8MinimalAPI/ApiBootstrap/ApiBootstrap.csproj b/src/NET8MinimalAPI/ApiBootstrap/ApiBootstrap.csproj index cf7e8a5..c3879a4 100644 --- a/src/NET8MinimalAPI/ApiBootstrap/ApiBootstrap.csproj +++ b/src/NET8MinimalAPI/ApiBootstrap/ApiBootstrap.csproj @@ -5,7 +5,7 @@ net8.0 true Lambda - bootstrap + bootstrap true false true @@ -15,14 +15,14 @@ - - - - - - - - + + + + + + + + diff --git a/src/NET8MinimalAPI/ApiBootstrap/CloudWatchQueryExecution.cs b/src/NET8MinimalAPI/ApiBootstrap/CloudWatchQueryExecution.cs deleted file mode 100644 index 23b847f..0000000 --- a/src/NET8MinimalAPI/ApiBootstrap/CloudWatchQueryExecution.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Amazon.CloudWatchLogs; -using Amazon.CloudWatchLogs.Model; -using Amazon.Lambda.Core; - -namespace GetProducts; - -public static class CloudWatchQueryExecution -{ - public static async Task>> RunQuery(AmazonCloudWatchLogsClient cloudWatchLogsClient) - { - var logGroupNamePrefix = - $"{Environment.GetEnvironmentVariable("LOG_GROUP_PREFIX")}{Environment.GetEnvironmentVariable("LAMBDA_ARCHITECTURE")}" - .Replace("_", "-"); - - var logGroupList = await cloudWatchLogsClient.DescribeLogGroupsAsync(new DescribeLogGroupsRequest() - { - LogGroupNamePrefix = logGroupNamePrefix, - }); - - var queryRes = await cloudWatchLogsClient.StartQueryAsync(new StartQueryRequest() - { - LogGroupNames = logGroupList.LogGroups.Select(p => p.LogGroupName).ToList(), - QueryString = - "filter @type=\"REPORT\" | fields greatest(@initDuration, 0) + @duration as duration, ispresent(@initDuration) as coldstart | stats count(*) as count, pct(duration, 50) as p50, pct(duration, 90) as p90, pct(duration, 99) as p99, max(duration) as max by coldstart", - StartTime = DateTime.Now.AddMinutes(-20).AsUnixTimestamp(), - EndTime = DateTime.Now.AsUnixTimestamp(), - }); - - QueryStatus currentQueryStatus = QueryStatus.Running; - List> finalResults = new List>(); - - while (currentQueryStatus == QueryStatus.Running || currentQueryStatus == QueryStatus.Scheduled) - { - var queryResults = await cloudWatchLogsClient.GetQueryResultsAsync(new GetQueryResultsRequest() - { - QueryId = queryRes.QueryId - }); - - currentQueryStatus = queryResults.Status; - finalResults = queryResults.Results; - - await Task.Delay(TimeSpan.FromSeconds(5)); - } - - return finalResults; - } -} \ No newline at end of file diff --git a/src/NET8MinimalAPI/ApiBootstrap/DateUtils.cs b/src/NET8MinimalAPI/ApiBootstrap/DateUtils.cs deleted file mode 100644 index 8a572e0..0000000 --- a/src/NET8MinimalAPI/ApiBootstrap/DateUtils.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; - -namespace GetProducts; - -public static class DateUtils -{ - public static long AsUnixTimestamp(this DateTime date) - { - DateTime origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); - TimeSpan diff = date.ToUniversalTime() - origin; - return (long)Math.Floor(diff.TotalSeconds); - } -} \ No newline at end of file diff --git a/src/NET8MinimalAPI/ApiBootstrap/Function.cs b/src/NET8MinimalAPI/ApiBootstrap/Function.cs index a4cd497..10a82a7 100644 --- a/src/NET8MinimalAPI/ApiBootstrap/Function.cs +++ b/src/NET8MinimalAPI/ApiBootstrap/Function.cs @@ -4,7 +4,6 @@ using System.Text.Json; using Amazon.CloudWatchLogs; using Amazon.CloudWatchLogs.Model; -using GetProducts; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; @@ -14,6 +13,15 @@ using Shared.Models; var app = Startup.Build(args); +var basePath=Environment.GetEnvironmentVariable("DEMO_BASE_PATH"); +if(String.IsNullOrEmpty(basePath)) + app.Logger.LogInformation($"No BASE PATH specified"); +else +{ + app.Logger.LogInformation($"Using BASE PATH:{basePath}"); + app.UsePathBase(basePath); + app.UseRouting(); +} var dataAccess = app.Services.GetRequiredService(); @@ -37,7 +45,7 @@ { var id = context.Request.RouteValues["id"].ToString(); - app.Logger.LogInformation($"Received request to delete {id}"); + app.Logger.LogInformation($"Received request to delete {id} from the database"); var product = await dataAccess.GetProduct(id); @@ -123,45 +131,4 @@ await context.Response.WriteAsJsonAsync(product); }); -app.MapGet("/test-results", async (HttpContext context) => -{ - var resultRows = 0; - var queryCount = 0; - - List> finalResults = new List>(); - - while (resultRows < 2 || queryCount >= 3) - { - finalResults = await CloudWatchQueryExecution.RunQuery(cloudWatchClient); - - resultRows = finalResults.Count; - queryCount++; - } - - var wrapper = new QueryResultWrapper() - { - LoadTestType = - $"{Environment.GetEnvironmentVariable("LOAD_TEST_TYPE")} ({Environment.GetEnvironmentVariable("LAMBDA_ARCHITECTURE")})", - WarmStart = new QueryResult() - { - Count = finalResults[0][1].Value, - P50 = finalResults[0][2].Value, - P90 = finalResults[0][3].Value, - P99 = finalResults[0][4].Value, - Max = finalResults[0][5].Value, - }, - ColdStart = new QueryResult() - { - Count = finalResults[1][1].Value, - P50 = finalResults[1][2].Value, - P90 = finalResults[1][3].Value, - P99 = finalResults[1][4].Value, - Max = finalResults[1][5].Value, - } - }; - - context.Response.StatusCode = (int) HttpStatusCode.OK; - await context.Response.WriteAsync(wrapper.AsMarkdownTableRow()); -}); - app.Run(); \ No newline at end of file diff --git a/src/NET8MinimalAPI/ApiBootstrap/QueryResult.cs b/src/NET8MinimalAPI/ApiBootstrap/QueryResult.cs deleted file mode 100644 index ff692d2..0000000 --- a/src/NET8MinimalAPI/ApiBootstrap/QueryResult.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace GetProducts; - -public record QueryResultWrapper -{ - public string LoadTestType { get; set; } - - public QueryResult ColdStart { get; set; } - - public QueryResult WarmStart { get; set; } - - public string AsMarkdownTableRow() => $"
Cold Start (ms)Warm Start (ms)
p50p90p99maxp50p90p99max
{LoadTestType}{ColdStart.P50}{ColdStart.P90}{ColdStart.P99}{ColdStart.Max}{WarmStart.P50}{WarmStart.P90}{WarmStart.P99}{WarmStart.Max}
"; -} - -public record QueryResult -{ - public string Count { get; set; } - - public string P50 { get; set; } - - public string P90 { get; set; } - - public string P99 { get; set; } - - public string Max { get; set; } -} \ No newline at end of file diff --git a/src/NET8MinimalAPI/Makefile b/src/NET8MinimalAPI/Makefile index 3f5ee87..ae88de0 100644 --- a/src/NET8MinimalAPI/Makefile +++ b/src/NET8MinimalAPI/Makefile @@ -1,3 +1,4 @@ -build-ApiFunction: - dotnet clean +build-MinimalApiX86: dotnet publish ApiBootstrap/ApiBootstrap.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) +build-MinimalApiArm64: + dotnet publish ApiBootstrap/ApiBootstrap.csproj -c Release -r linux-arm64 --self-contained -o $(ARTIFACTS_DIR) \ No newline at end of file diff --git a/src/NET8MinimalAPI/Shared/Shared.csproj b/src/NET8MinimalAPI/Shared/Shared.csproj index 34cd820..7d6a8a2 100644 --- a/src/NET8MinimalAPI/Shared/Shared.csproj +++ b/src/NET8MinimalAPI/Shared/Shared.csproj @@ -1,14 +1,14 @@ - net8.0 + net8.0 enable enable - - + + diff --git a/src/NET8MinimalAPI/deploy.sh b/src/NET8MinimalAPI/deploy.sh new file mode 100755 index 0000000..1caf08f --- /dev/null +++ b/src/NET8MinimalAPI/deploy.sh @@ -0,0 +1,43 @@ +#Arguments: +#$1 - delete stack formation to ensure that all test have same conditon and favour similar ammount of cold start events + +STACK_NAME=dotnet8-minimal-api +DELETE_STACK=yes + +COLOR='\033[0;33m' +NO_COLOR='\033[0m' # No Color + +if [ "x$1" != x ]; +then + DELETE_STACK=$1 +fi + +echo "${COLOR}" +echo -------------------------------------------- +echo DELETE_STACK: $DELETE_STACK +echo -------------------------------------------- +echo "${NO_COLOR}" + +if [ $DELETE_STACK == "yes" ]; +then + echo "${COLOR}" + echo -------------------------------------------- + echo DELETING STACK $STACK_NAME + echo -------------------------------------------- + echo "${NO_COLOR}" + aws cloudformation delete-stack --stack-name $STACK_NAME + echo "${COLOR}" + echo --------------------------------------------- + echo Waiting stack to be deleted + echo -------------------------------------------- + echo "${NO_COLOR}" + aws cloudformation wait stack-delete-complete --stack-name $STACK_NAME + echo "${COLOR}" + echo --------------------------------------------- + echo Stack deleted + echo -------------------------------------------- + echo "${NO_COLOR}" +fi + +sam build +sam deploy --stack-name dotnet8-minimal-api --resolve-s3 --s3-prefix dotnet8-minimal-api --no-confirm-changeset --no-fail-on-empty-changeset --capabilities CAPABILITY_IAM \ No newline at end of file diff --git a/src/NET8MinimalAPI/omnisharp.json b/src/NET8MinimalAPI/omnisharp.json deleted file mode 100644 index c42f8db..0000000 --- a/src/NET8MinimalAPI/omnisharp.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "fileOptions": { - "excludeSearchPatterns": [ - "**/bin/**/*", - "**/obj/**/*" - ] - }, - "msbuild": { - "Platform": "rhel.7.2-x64" - } -} \ No newline at end of file diff --git a/src/NET8MinimalAPI/run-loadtest.sh b/src/NET8MinimalAPI/run-loadtest.sh new file mode 100755 index 0000000..c8ee870 --- /dev/null +++ b/src/NET8MinimalAPI/run-loadtest.sh @@ -0,0 +1,150 @@ +#Arguments: +#$1 - load test duration in seconds +#$2 - log interval to be used in the cloudwatch query in minutes +#$3 - when equal to yes cloudwatch log group will be deleted to ensure that only logs of the load test will be evaluated for stat +#$4 - ARN of sns topic to notify test results + +STACK_NAME=dotnet8-minimal-api +TEST_DURATIOMN_SEC=60 +LOG_INTERVAL_MIN=20 +LOG_DELETE=yes + +COLOR='\033[0;33m' +NO_COLOR='\033[0m' # No Color + +if [ "x$1" != x ]; +then + TEST_DURATIOMN_SEC=$1 +fi + +if [ "x$2" != x ]; +then + LOG_INTERVAL_MIN=$2 +fi + +if [ "x$3" != x ]; +then + LOG_DELETE=$3 +fi + +if [ "x$4" != x ]; +then + SNS_TOPIC_ARN=$4 +fi + +echo "${COLOR}" +echo -------------------------------------------- +echo DURATION:$TEST_DURATIOMN_SEC +echo LOG INTERVAL:$LOG_INTERVAL_MIN +echo LOG_DELETE: $LOG_DELETE +echo SNS_TOPIC_ARN: $SNS_TOPIC_ARN +echo -------------------------------------------- +echo "${NO_COLOR}" + +mkdir -p Report + +function RunLoadTest() +{ + #Params: + #$1 - Architecture (x86 or arm64).Used for logging and naming report file + #$2 - Stack output name to get API Url + #$3 - Stack output name to get lambda name + + #get test params from cloud formation output + echo "${COLOR}" + export API_URL=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$2'].OutputValue" \ + --output text) + echo API URL: $API_URL + + LAMBDA=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$3'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA + + if [ $LOG_DELETE == "yes" ]; + then + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA + echo --------------------------------------------- + echo Waiting 10 sec. for deletion to complete + echo -------------------------------------------- + sleep 10 + fi + + #run load test with artillery + echo -------------------------------------------- + echo $1 RUNNING LOAD TEST $TEST_DURATIOMN_SEC sec $LAMBDA: $API_URL + echo -------------------------------------------- + echo "${NO_COLOR}" + artillery run \ + --overrides '{"config": { "phases": [{ "duration": '$TEST_DURATIOMN_SEC', "arrivalRate": 100 }] } }' \ + --quiet \ + ../../loadtest/codebuild/load-test.yml + + echo "${COLOR}" + echo -------------------------------------------- + echo Waiting 10 sec. for logs to consolidate + echo -------------------------------------------- + sleep 10 + + #get stats from cloudwatch + enddate=$(date "+%s") + startdate=$(($enddate-($LOG_INTERVAL_MIN*60))) + echo -------------------------------------------- + echo Log start:$startdate end:$enddate + echo -------------------------------------------- + + echo -------------------------------------------- + echo GET ERROR METRICS $1 + echo -------------------------------------------- + aws cloudwatch get-metric-statistics \ + --namespace AWS/Lambda \ + --metric-name Errors \ + --dimensions Name=FunctionName,Value=$LAMBDA \ + --statistics Sum --period 43200 \ + --start-time $startdate --end-time $enddate > ./Report/load-test-errors-$1.json + + QUERY_ID=$(aws logs start-query \ + --log-group-name /aws/lambda/$LAMBDA \ + --start-time $startdate \ + --end-time $enddate \ + --query-string 'filter @type="REPORT" | fields greatest(@initDuration, 0) + @duration as duration, ispresent(@initDuration) as coldstart | stats count(*) as count, pct(duration, 50) as p50, pct(duration, 90) as p90, pct(duration, 99) as p99, max(duration) as max by coldstart' \ + | jq -r '.queryId') + + echo -------------------------------------------- + echo Query started, id: $QUERY_ID + echo -------------------------------------------- + + echo --------------------------------------------- + echo Waiting 10 sec. for cloudwatch query to complete + echo -------------------------------------------- + sleep 10 + + echo -------------------------------------------- + echo RESULTS $LAMBDA + echo -------------------------------------------- + echo "${NO_COLOR}" + date > ./Report/load-test-report-$1.txt + echo $1 RESULTS lambda: $LAMBDA >> ./Report/load-test-report-$1.txt + echo Test duration sec: $TEST_DURATIOMN_SEC >> ./Report/load-test-report-$1.txt + echo Log interval min: $LOG_INTERVAL_MIN >> ./Report/load-test-report-$1.txt + aws logs get-query-results --query-id $QUERY_ID --output text >> ./Report/load-test-report-$1.txt + cat ./Report/load-test-report-$1.txt + aws logs get-query-results --query-id $QUERY_ID --output json >> ./Report/load-test-report-$1.json + + if [ "x$SNS_TOPIC_ARN" != x ]; + then + echo -------------------------------------------- + echo Sending message to sns topic: $SNS_TOPIC_ARN + echo -------------------------------------------- + msg=$(<./Report/load-test-report-$1.txt)\n\n$(<./Report/load-test-errors-$1.json) + subject="serverless dotnet demo load test result for $LAMBDA" + aws sns publish --topic-arn $SNS_TOPIC_ARN --subject "$subject" --message "$msg" + fi +} + +RunLoadTest x86 ApiUrlX86 LambdaX86Name +RunLoadTest arm64 ApiUrlArm64 LambdaArm64Name \ No newline at end of file diff --git a/src/NET8MinimalAPI/template.yaml b/src/NET8MinimalAPI/template.yaml index aeb8b06..abb40f5 100644 --- a/src/NET8MinimalAPI/template.yaml +++ b/src/NET8MinimalAPI/template.yaml @@ -1,47 +1,69 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 +Parameters: + x86FunctionName: + Type: String + Default: Net8-MinimalApi-X86 + arm64FunctionName: + Type: String + Default: Net8-MinimalApi-Arm64 + Globals: Function: MemorySize: 1024 - Architectures: ["x86_64"] Runtime: provided.al2 Timeout: 30 Tracing: Active Environment: Variables: PRODUCT_TABLE_NAME: !Ref Table - LOG_GROUP_PREFIX: !Sub "/aws/lambda/net-8-minimal-" - LOAD_TEST_TYPE: "NET 8 Minimal API" Resources: - ApiFunction: + MinimalApiX86: + Type: AWS::Serverless::Function + Properties: + FunctionName: !Ref x86FunctionName + Architectures: [x86_64] + CodeUri: ./ + Handler: ApiBootstrap + Environment: + Variables: + DEMO_BASE_PATH: /x86 + Events: + Api: + Type: HttpApi + Properties: + Path: /x86/{proxy+} + Method: ANY + Policies: + - DynamoDBCrudPolicy: + TableName: + !Ref Table + Metadata: + BuildMethod: makefile + MinimalApiArm64: Type: AWS::Serverless::Function Properties: + FunctionName: !Ref arm64FunctionName + Architectures: [arm64] CodeUri: ./ Handler: ApiBootstrap + Environment: + Variables: + DEMO_BASE_PATH: /arm64 Events: Api: Type: HttpApi Properties: - Path: /{proxy+} + Path: /arm64/{proxy+} Method: ANY Policies: - DynamoDBCrudPolicy: TableName: !Ref Table - - Version: "2012-10-17" - Statement: - - Sid: AllowStartQueries - Effect: Allow - Action: - - logs:DescribeLogGroups - - logs:StartQuery - Resource: "*" - - Sid: AllowGetQueryResults - Effect: Allow - Action: logs:GetQueryResults - Resource: "*" + Metadata: + BuildMethod: makefile Table: Type: AWS::DynamoDB::Table @@ -59,4 +81,16 @@ Resources: Outputs: ApiUrl: Description: "API Gateway endpoint URL" - Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/" \ No newline at end of file + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com" + LambdaX86Name: + Description: "Lambda X86 Name" + Value: !Ref x86FunctionName + ApiUrlX86: + Description: "X86 API endpoint URL" + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/x86" + LambdaArm64Name: + Description: "Lambda Arm64 Name" + Value: !Ref arm64FunctionName + ApiUrlArm64: + Description: "Arm64 GateAPI endpoint URL" + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/arm64" \ No newline at end of file diff --git a/src/NET8Native/DeleteProduct/DeleteProduct.csproj b/src/NET8Native/DeleteProduct/DeleteProduct.csproj index 1be8350..9f5a81b 100644 --- a/src/NET8Native/DeleteProduct/DeleteProduct.csproj +++ b/src/NET8Native/DeleteProduct/DeleteProduct.csproj @@ -3,25 +3,29 @@ exe net8.0 - true - Lambda bootstrap + true - false - true - true + true + Lambda + true + skylake Speed + + false + true + true - - - - - - + + + + + + diff --git a/src/NET8Native/DeleteProduct/aws-lambda-tools-defaults.json b/src/NET8Native/DeleteProduct/aws-lambda-tools-defaults.json deleted file mode 100644 index 1d112e0..0000000 --- a/src/NET8Native/DeleteProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET8Native/GenerateLoadTestResults/DateUtils.cs b/src/NET8Native/GenerateLoadTestResults/DateUtils.cs deleted file mode 100644 index c37c43d..0000000 --- a/src/NET8Native/GenerateLoadTestResults/DateUtils.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; - -namespace GenerateLoadTestResults; - -public static class DateUtils -{ - public static long AsUnixTimestamp(this DateTime date) - { - DateTime origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); - TimeSpan diff = date.ToUniversalTime() - origin; - return (long)Math.Floor(diff.TotalSeconds); - } -} \ No newline at end of file diff --git a/src/NET8Native/GenerateLoadTestResults/Function.cs b/src/NET8Native/GenerateLoadTestResults/Function.cs deleted file mode 100644 index 21a57a8..0000000 --- a/src/NET8Native/GenerateLoadTestResults/Function.cs +++ /dev/null @@ -1,146 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text.Json; -using System.Threading.Tasks; -using Amazon.CloudWatchLogs; -using Amazon.CloudWatchLogs.Model; -using Amazon.Lambda.APIGatewayEvents; -using Amazon.Lambda.Core; - -[assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.SystemTextJson.DefaultLambdaJsonSerializer))] - -namespace GenerateLoadTestResults -{ - public class Function - { - private AmazonCloudWatchLogsClient _cloudWatchLogsClient; - - public Function() - { - this._cloudWatchLogsClient = new AmazonCloudWatchLogsClient(); - } - - public async Task FunctionHandler( - APIGatewayHttpApiV2ProxyRequest apigProxyEvent, - ILambdaContext context) - { - if (!apigProxyEvent.RequestContext.Http.Method.Equals(HttpMethod.Get.Method)) - { - return new APIGatewayHttpApiV2ProxyResponse - { - Body = "Only GET allowed", - StatusCode = (int) HttpStatusCode.MethodNotAllowed, - }; - } - - try - { - var resultRows = 0; - var queryCount = 0; - - List> finalResults = new List>(); - - while (resultRows < 2 || queryCount >= 3) - { - finalResults = await runQuery(context); - - resultRows = finalResults.Count; - queryCount++; - } - - var wrapper = new QueryResultWrapper() - { - LoadTestType = - $"{Environment.GetEnvironmentVariable("LOAD_TEST_TYPE")} ({Environment.GetEnvironmentVariable("LAMBDA_ARCHITECTURE")})", - WarmStart = new QueryResult() - { - Count = finalResults[0][1].Value, - P50 = finalResults[0][2].Value, - P90 = finalResults[0][3].Value, - P99 = finalResults[0][4].Value, - Max = finalResults[0][5].Value, - }, - ColdStart = new QueryResult() - { - Count = finalResults[1][1].Value, - P50 = finalResults[1][2].Value, - P90 = finalResults[1][3].Value, - P99 = finalResults[1][4].Value, - Max = finalResults[1][5].Value, - } - }; - - return new APIGatewayHttpApiV2ProxyResponse - { - StatusCode = (int) HttpStatusCode.OK, - Body = wrapper.AsMarkdownTableRow(), - Headers = new Dictionary {{"Content-Type", "text/html"}} - }; - } - catch (Exception e) - { - context.Logger.LogLine($"Error retrieving results {e.Message} {e.StackTrace}"); - - return new APIGatewayHttpApiV2ProxyResponse - { - Body = "Not Found", - StatusCode = (int) HttpStatusCode.InternalServerError, - }; - } - } - - private async Task>> runQuery(ILambdaContext context) - { - var logGroupNamePrefix = - $"{Environment.GetEnvironmentVariable("LOG_GROUP_PREFIX")}{Environment.GetEnvironmentVariable("LAMBDA_ARCHITECTURE")}" - .Replace("_", "-"); - - context.Logger.LogLine($"Retrieving log groups with prefix {logGroupNamePrefix}"); - - var logGroupList = await _cloudWatchLogsClient.DescribeLogGroupsAsync(new DescribeLogGroupsRequest() - { - LogGroupNamePrefix = logGroupNamePrefix, - }); - - context.Logger.LogLine($"Found {logGroupList.LogGroups.Count} log group(s)"); - - var queryRes = await _cloudWatchLogsClient.StartQueryAsync(new StartQueryRequest() - { - LogGroupNames = logGroupList.LogGroups.Select(p => p.LogGroupName).ToList(), - QueryString = - "filter @type=\"REPORT\" | fields greatest(@initDuration, 0) + @duration as duration, ispresent(@initDuration) as coldstart | stats count(*) as count, pct(duration, 50) as p50, pct(duration, 90) as p90, pct(duration, 99) as p99, max(duration) as max by coldstart", - StartTime = DateTime.Now.AddMinutes(-20).AsUnixTimestamp(), - EndTime = DateTime.Now.AsUnixTimestamp(), - }); - - context.Logger.LogLine($"Running query, query id is {queryRes.QueryId}"); - - QueryStatus currentQueryStatus = QueryStatus.Running; - List> finalResults = new List>(); - - while (currentQueryStatus == QueryStatus.Running || currentQueryStatus == QueryStatus.Scheduled) - { - context.Logger.LogLine("Retrieving query results"); - - var queryResults = await _cloudWatchLogsClient.GetQueryResultsAsync(new GetQueryResultsRequest() - { - QueryId = queryRes.QueryId - }); - - context.Logger.LogLine($"Query result status is {queryResults.Status}"); - - currentQueryStatus = queryResults.Status; - finalResults = queryResults.Results; - - await Task.Delay(TimeSpan.FromSeconds(5)); - } - - context.Logger.LogLine($"Final results: {finalResults.Count} row(s)"); - - return finalResults; - } - } -} \ No newline at end of file diff --git a/src/NET8Native/GenerateLoadTestResults/GenerateLoadTestResults.csproj b/src/NET8Native/GenerateLoadTestResults/GenerateLoadTestResults.csproj deleted file mode 100644 index 4a8d708..0000000 --- a/src/NET8Native/GenerateLoadTestResults/GenerateLoadTestResults.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - net6.0 - true - true - - - - - - - - - - - diff --git a/src/NET8Native/GenerateLoadTestResults/QueryResult.cs b/src/NET8Native/GenerateLoadTestResults/QueryResult.cs deleted file mode 100644 index 69b5a31..0000000 --- a/src/NET8Native/GenerateLoadTestResults/QueryResult.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace GenerateLoadTestResults; - -public record QueryResultWrapper -{ - public string LoadTestType { get; set; } - - public QueryResult ColdStart { get; set; } - - public QueryResult WarmStart { get; set; } - - public string AsMarkdownTableRow() => $"
Cold Start (ms)Warm Start (ms)
p50p90p99maxp50p90p99max
{LoadTestType}{ColdStart.P50}{ColdStart.P90}{ColdStart.P99}{ColdStart.Max}{WarmStart.P50}{WarmStart.P90}{WarmStart.P99}{WarmStart.Max}
"; -} - -public record QueryResult -{ - public string Count { get; set; } - - public string P50 { get; set; } - - public string P90 { get; set; } - - public string P99 { get; set; } - - public string Max { get; set; } -} \ No newline at end of file diff --git a/src/NET8Native/GetProduct/GetProduct.csproj b/src/NET8Native/GetProduct/GetProduct.csproj index 1354b30..364daed 100644 --- a/src/NET8Native/GetProduct/GetProduct.csproj +++ b/src/NET8Native/GetProduct/GetProduct.csproj @@ -1,29 +1,29 @@  - exe net8.0 - true - Lambda bootstrap true - false - true - true - + true + Lambda + true + skylake Speed + + false + true + true - - - - - - + + + + + + diff --git a/src/NET8Native/GetProduct/aws-lambda-tools-defaults.json b/src/NET8Native/GetProduct/aws-lambda-tools-defaults.json deleted file mode 100644 index 1d112e0..0000000 --- a/src/NET8Native/GetProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET8Native/GetProducts/GetProducts.csproj b/src/NET8Native/GetProducts/GetProducts.csproj index 578c023..364daed 100644 --- a/src/NET8Native/GetProducts/GetProducts.csproj +++ b/src/NET8Native/GetProducts/GetProducts.csproj @@ -1,26 +1,29 @@  - exe net8.0 - true - Lambda bootstrap + true - false - true - true + true + Lambda + true + skylake Speed + + false + true + true - - - - - - + + + + + + diff --git a/src/NET8Native/GetProducts/aws-lambda-tools-defaults.json b/src/NET8Native/GetProducts/aws-lambda-tools-defaults.json deleted file mode 100644 index 1d112e0..0000000 --- a/src/NET8Native/GetProducts/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Information" : [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - - "dotnet lambda help", - - "All the command line options for the Lambda command can be specified in this file." - ], - - "profile":"", - "region" : "", - "configuration": "Release", - "function-runtime":"provided.al2", - "function-memory-size" : 256, - "function-timeout" : 30, - "function-handler" : "CancelBooking" -} diff --git a/src/NET8Native/Makefile b/src/NET8Native/Makefile index 9cc4200..9993dc3 100644 --- a/src/NET8Native/Makefile +++ b/src/NET8Native/Makefile @@ -1,11 +1,11 @@ -build-GetProductsFunction: - dotnet publish -c Release -r linux-x64 ./GetProducts/GetProducts.csproj -o $(ARTIFACTS_DIR) +build-GetProductFunctionX86: + dotnet publish GetProduct/GetProduct.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) -build-GetProductFunction: - dotnet publish -c Release -r linux-x64 ./GetProduct/GetProduct.csproj -o $(ARTIFACTS_DIR) +build-GetProductsFunctionX86: + dotnet publish GetProducts/GetProducts.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) -build-DeleteProductFunction: - dotnet publish -c Release -r linux-x64 ./DeleteProduct/DeleteProduct.csproj -o $(ARTIFACTS_DIR) +build-PutProductFunctionX86: + dotnet publish PutProduct/PutProduct.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) -build-PutProductFunction: - dotnet publish -c Release -r linux-x64 ./PutProduct/PutProduct.csproj -o $(ARTIFACTS_DIR) \ No newline at end of file +build-DeleteProductFunctionX86: + dotnet publish DeleteProduct/DeleteProduct.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) \ No newline at end of file diff --git a/src/NET8Native/PutProduct/PutProduct.csproj b/src/NET8Native/PutProduct/PutProduct.csproj index 1be8350..9f5a81b 100644 --- a/src/NET8Native/PutProduct/PutProduct.csproj +++ b/src/NET8Native/PutProduct/PutProduct.csproj @@ -3,25 +3,29 @@ exe net8.0 - true - Lambda bootstrap + true - false - true - true + true + Lambda + true + skylake Speed + + false + true + true - - - - - - + + + + + + diff --git a/src/NET8Native/PutProduct/aws-lambda-tools-defaults.json b/src/NET8Native/PutProduct/aws-lambda-tools-defaults.json deleted file mode 100644 index 7ed1688..0000000 --- a/src/NET8Native/PutProduct/aws-lambda-tools-defaults.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Information": [ - "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.", - "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.", - "dotnet lambda help", - "All the command line options for the Lambda command can be specified in this file." - ], - "profile": "", - "region": "", - "configuration": "Release", - "function-runtime": "provided.al2", - "function-memory-size": 256, - "function-timeout": 30, - "function-handler": "bootstrap" -} \ No newline at end of file diff --git a/src/NET8Native/Shared/JsonSerializerContext.cs b/src/NET8Native/Shared/JsonSerializerContext.cs index 99f168f..f71528f 100644 --- a/src/NET8Native/Shared/JsonSerializerContext.cs +++ b/src/NET8Native/Shared/JsonSerializerContext.cs @@ -1,4 +1,5 @@ -using System.Text.Json.Serialization; +using System.Collections.Generic; +using System.Text.Json.Serialization; using Amazon.Lambda.APIGatewayEvents; using Shared.Models; diff --git a/src/NET8Native/Shared/Shared.csproj b/src/NET8Native/Shared/Shared.csproj index a2b16db..84e7fb3 100644 --- a/src/NET8Native/Shared/Shared.csproj +++ b/src/NET8Native/Shared/Shared.csproj @@ -2,26 +2,25 @@ net8.0 - enable - enable - true - true - false - true - true - + true + Lambda + true + skylake Speed + + false + true + true - - - - + + + + diff --git a/src/NET8Native/deploy.sh b/src/NET8Native/deploy.sh new file mode 100755 index 0000000..a6d0c85 --- /dev/null +++ b/src/NET8Native/deploy.sh @@ -0,0 +1,43 @@ +#Arguments: +#$1 - delete stack formation to ensure that all test have same conditon and favour similar ammount of cold start events + +STACK_NAME=dotnet8-native +DELETE_STACK=yes + +COLOR='\033[0;33m' +NO_COLOR='\033[0m' # No Color + +if [ "x$1" != x ]; +then + DELETE_STACK=$1 +fi + +echo "${COLOR}" +echo -------------------------------------------- +echo DELETE_STACK: $DELETE_STACK +echo -------------------------------------------- +echo "${NO_COLOR}" + +if [ $DELETE_STACK == "yes" ]; +then + echo "${COLOR}" + echo -------------------------------------------- + echo DELETING STACK $STACK_NAME + echo -------------------------------------------- + echo "${NO_COLOR}" + aws cloudformation delete-stack --stack-name $STACK_NAME + echo "${COLOR}" + echo --------------------------------------------- + echo Waiting stack to be deleted + echo -------------------------------------------- + echo "${NO_COLOR}" + aws cloudformation wait stack-delete-complete --stack-name $STACK_NAME + echo "${COLOR}" + echo --------------------------------------------- + echo Stack deleted + echo -------------------------------------------- + echo "${NO_COLOR}" +fi + +sam build --use-container --build-image plantpowerjames/dotnet-8-lambda-build:rc2 +sam deploy --stack-name $STACK_NAME --resolve-s3 --s3-prefix $STACK_NAME --no-confirm-changeset --no-fail-on-empty-changeset --capabilities CAPABILITY_IAM \ No newline at end of file diff --git a/src/NET8Native/run-loadtest.sh b/src/NET8Native/run-loadtest.sh new file mode 100755 index 0000000..48bc775 --- /dev/null +++ b/src/NET8Native/run-loadtest.sh @@ -0,0 +1,181 @@ +#Arguments: +#$1 - load test duration in seconds +#$2 - log interval to be used in the cloudwatch query in minutes +#$3 - when equal to yes cloudwatch log group will be deleted to ensure that only logs of the load test will be evaluated for stat +#$4 - ARN of sns topic to notify test results + +STACK_NAME=dotnet8-native +TEST_DURATIOMN_SEC=60 +LOG_INTERVAL_MIN=20 +LOG_DELETE=yes + +COLOR='\033[0;33m' +NO_COLOR='\033[0m' # No Color + +if [ "x$1" != x ]; +then + TEST_DURATIOMN_SEC=$1 +fi + +if [ "x$2" != x ]; +then + LOG_INTERVAL_MIN=$2 +fi + +if [ "x$3" != x ]; +then + LOG_DELETE=$3 +fi + +if [ "x$4" != x ]; +then + SNS_TOPIC_ARN=$4 +fi + +echo "${COLOR}" +echo -------------------------------------------- +echo DURATION:$TEST_DURATIOMN_SEC +echo LOG INTERVAL:$LOG_INTERVAL_MIN +echo LOG_DELETE: $LOG_DELETE +echo SNS_TOPIC_ARN: $SNS_TOPIC_ARN +echo -------------------------------------------- +echo "${NO_COLOR}" + +mkdir -p Report + +function RunLoadTest() +{ + #Params: + #$1 - Architecture (x86 or arm64).Used for logging and naming report file + #$2 - Stack output name to get API Url + #$3 - Stack output name to get lambda name GetProducts + #$4 - Stack output name to get lambda name GetProduct + #$5 - Stack output name to get lambda name DeleteProduct + #$6 - Stack output name to get lambda name PutProduct + + #get test params from cloud formation output + echo "${COLOR}" + export API_URL=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$2'].OutputValue" \ + --output text) + echo API URL: $API_URL + + LAMBDA_GETPRODUCTS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$3'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA_GETPRODUCTS + + LAMBDA_GETPRODUCT=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$4'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA_GETPRODUCT + + LAMBDA_DELETEPRODUCT=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$5'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA_DELETEPRODUCT + + LAMBDA_PUTPRODUCT=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$6'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA_PUTPRODUCT + + if [ $LOG_DELETE == "yes" ]; + then + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA_GETPRODUCTS + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA_GETPRODUCTS + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA_GETPRODUCT + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA_GETPRODUCT + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA_DELETEPRODUCT + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA_DELETEPRODUCT + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA_PUTPRODUCT + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA_PUTPRODUCT + echo --------------------------------------------- + echo Waiting 10 sec. for deletion to complete + echo -------------------------------------------- + sleep 10 + fi + + #run load test with artillery + echo -------------------------------------------- + echo $1 RUNNING LOAD TEST $TEST_DURATIOMN_SEC sec $API_URL + echo -------------------------------------------- + echo "${NO_COLOR}" + artillery run \ + --overrides '{"config": { "phases": [{ "duration": '$TEST_DURATIOMN_SEC', "arrivalRate": 100 }] } }' \ + --quiet \ + ../../loadtest/codebuild/load-test.yml + + echo "${COLOR}" + echo -------------------------------------------- + echo Waiting 10 sec. for logs to consolidate + echo -------------------------------------------- + sleep 10 + + #get stats from cloudwatch + enddate=$(date "+%s") + startdate=$(($enddate-($LOG_INTERVAL_MIN*60))) + echo -------------------------------------------- + echo Log start:$startdate end:$enddate + echo -------------------------------------------- + + echo -------------------------------------------- + echo GET ERROR METRICS $1 + echo -------------------------------------------- + aws cloudwatch get-metric-statistics \ + --namespace AWS/Lambda \ + --metric-name Errors \ + --dimensions Name=FunctionName,Value=$LAMBDA_GETPRODUCTS Name=FunctionName,Value=$LAMBDA_GETPRODUCT Name=FunctionName,Value=$LAMBDA_DELETEPRODUCT Name=FunctionName,Value=$LAMBDA_PUTPRODUCT \ + --statistics Sum --period 43200 \ + --start-time $startdate --end-time $enddate > ./Report/load-test-errors-$1.json + + cat ./Report/load-test-errors-$1.json + + QUERY_ID=$(aws logs start-query \ + --log-group-names "/aws/lambda/$LAMBDA_GETPRODUCTS" "/aws/lambda/$LAMBDA_GETPRODUCT" "/aws/lambda/$LAMBDA_DELETEPRODUCT" "/aws/lambda/$LAMBDA_PUTPRODUCT" \ + --start-time $startdate \ + --end-time $enddate \ + --query-string 'filter @type="REPORT" | fields greatest(@initDuration, 0) + @duration as duration, ispresent(@initDuration) as coldstart | stats count(*) as count, pct(duration, 50) as p50, pct(duration, 90) as p90, pct(duration, 99) as p99, max(duration) as max by coldstart' \ + | jq -r '.queryId') + + echo -------------------------------------------- + echo Query started, id: $QUERY_ID + echo -------------------------------------------- + + echo --------------------------------------------- + echo Waiting 10 sec. for cloudwatch query to complete + echo -------------------------------------------- + sleep 10 + + echo -------------------------------------------- + echo RESULTS $1 + echo -------------------------------------------- + echo "${NO_COLOR}" + date > ./Report/load-test-report-$1.txt + echo $1 RESULTS >> ./Report/load-test-report-$1.txt + echo Test duration sec: $TEST_DURATIOMN_SEC >> ./Report/load-test-report-$1.txt + echo Log interval min: $LOG_INTERVAL_MIN >> ./Report/load-test-report-$1.txt + aws logs get-query-results --query-id $QUERY_ID --output text >> ./Report/load-test-report-$1.txt + cat ./Report/load-test-report-$1.txt + aws logs get-query-results --query-id $QUERY_ID --output json >> ./Report/load-test-report-$1.json + + if [ "x$SNS_TOPIC_ARN" != x ]; + then + echo -------------------------------------------- + echo Sending message to sns topic: $SNS_TOPIC_ARN + echo -------------------------------------------- + msg=$(<./Report/load-test-report-$1.txt)\n\n$(<./Report/load-test-errors-$1.json) + subject="serverless dotnet demo load test result for $LAMBDA_GETPRODUCTS" + aws sns publish --topic-arn $SNS_TOPIC_ARN --subject "$subject" --message "$msg" + fi +} + +RunLoadTest x86 ApiUrlX86 LambdaX86NameGetProducts LambdaX86NameGetProduct LambdaX86NameDeleteProduct LambdaX86NamePutProduct \ No newline at end of file diff --git a/src/NET8Native/template.yaml b/src/NET8Native/template.yaml index 22dc554..88b4d70 100644 --- a/src/NET8Native/template.yaml +++ b/src/NET8Native/template.yaml @@ -1,10 +1,14 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 +Parameters: + x86FunctionNamePrefix: + Type: String + Default: Net8-Native-X86 + Globals: Function: MemorySize: 1024 - Architectures: ["x86_64"] Runtime: provided.al2 Timeout: 30 Tracing: Active @@ -13,16 +17,19 @@ Globals: PRODUCT_TABLE_NAME: !Ref Table Resources: - GetProductsFunction: + #X86 + GetProductsFunctionX86: Type: AWS::Serverless::Function Properties: - CodeUri: . + FunctionName: !Join [ -,[!Ref x86FunctionNamePrefix,GetProducts]] + Architectures: [x86_64] + CodeUri: ./ Handler: bootstrap Events: Api: Type: HttpApi Properties: - Path: / + Path: /x86 Method: GET Policies: - DynamoDBReadPolicy: @@ -31,16 +38,18 @@ Resources: Metadata: BuildMethod: makefile - GetProductFunction: + GetProductFunctionX86: Type: AWS::Serverless::Function Properties: - CodeUri: . + FunctionName: !Join [ -,[!Ref x86FunctionNamePrefix,GetProduct]] + Architectures: [x86_64] + CodeUri: ./ Handler: GetProduct Events: Api: Type: HttpApi Properties: - Path: /{id} + Path: /x86/{id} Method: GET Policies: - Version: "2012-10-17" @@ -51,16 +60,18 @@ Resources: Metadata: BuildMethod: makefile - DeleteProductFunction: + DeleteProductFunctionX86: Type: AWS::Serverless::Function Properties: - CodeUri: . + FunctionName: !Join [ -,[!Ref x86FunctionNamePrefix,DeleteProduct]] + Architectures: [x86_64] + CodeUri: ./ Handler: DeleteProduct Events: Api: Type: HttpApi Properties: - Path: /{id} + Path: /x86/{id} Method: DELETE Policies: - Version: "2012-10-17" @@ -73,16 +84,18 @@ Resources: Metadata: BuildMethod: makefile - PutProductFunction: + PutProductFunctionX86: Type: AWS::Serverless::Function Properties: - CodeUri: . + FunctionName: !Join [ -,[!Ref x86FunctionNamePrefix,PutProduct]] + Architectures: [x86_64] + CodeUri: ./ Handler: PutProduct Events: Api: Type: HttpApi Properties: - Path: /{id} + Path: /x86/{id} Method: PUT Policies: - Version: "2012-10-17" @@ -107,4 +120,21 @@ Resources: Outputs: ApiUrl: Description: "API Gateway endpoint URL" - Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/" \ No newline at end of file + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com" + ApiUrlX86: + Description: "X86 API endpoint URL" + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/x86" + + #x86 + LambdaX86NameGetProducts: + Description: "Lambda X86 GetProducts" + Value: !Join [ -,[!Ref x86FunctionNamePrefix,GetProducts]] + LambdaX86NameGetProduct: + Description: "Lambda X86 GetProduct" + Value: !Join [ -,[!Ref x86FunctionNamePrefix,GetProduct]] + LambdaX86NameDeleteProduct: + Description: "Lambda X86 DeleteProduct" + Value: !Join [ -,[!Ref x86FunctionNamePrefix,DeleteProduct]] + LambdaX86NamePutProduct: + Description: "Lambda X86 PutProduct" + Value: !Join [ -,[!Ref x86FunctionNamePrefix,PutProduct]] \ No newline at end of file diff --git a/src/NET8NativeMinimalAPI/ApiBootstrap/ApiBootstrap.csproj b/src/NET8NativeMinimalAPI/ApiBootstrap/ApiBootstrap.csproj index 48bb4ef..e2c8e51 100644 --- a/src/NET8NativeMinimalAPI/ApiBootstrap/ApiBootstrap.csproj +++ b/src/NET8NativeMinimalAPI/ApiBootstrap/ApiBootstrap.csproj @@ -3,24 +3,28 @@ Exe net8.0 + bootstrap + + true + + true true enable enable GetProducts - true - bootstrap + full - true - - - - - - - + + + + + + + + @@ -28,6 +32,8 @@ + + diff --git a/src/NET8NativeMinimalAPI/Makefile b/src/NET8NativeMinimalAPI/Makefile index fc76f1e..0e8e9c9 100644 --- a/src/NET8NativeMinimalAPI/Makefile +++ b/src/NET8NativeMinimalAPI/Makefile @@ -1,2 +1,2 @@ -build-ApiFunction: - dotnet publish -c Release -r linux-x64 ./ApiBootstrap/ApiBootstrap.csproj -o $(ARTIFACTS_DIR) \ No newline at end of file +build-MinimalApiX86: + dotnet publish ApiBootstrap/ApiBootstrap.csproj -c Release -r linux-x64 --self-contained -o $(ARTIFACTS_DIR) \ No newline at end of file diff --git a/src/NET8NativeMinimalAPI/Shared/Shared.csproj b/src/NET8NativeMinimalAPI/Shared/Shared.csproj index 8361c85..466b824 100644 --- a/src/NET8NativeMinimalAPI/Shared/Shared.csproj +++ b/src/NET8NativeMinimalAPI/Shared/Shared.csproj @@ -2,13 +2,17 @@ net8.0 + + true + + true enable enable - - + + diff --git a/src/NET8NativeMinimalAPI/deploy.sh b/src/NET8NativeMinimalAPI/deploy.sh new file mode 100755 index 0000000..d6da858 --- /dev/null +++ b/src/NET8NativeMinimalAPI/deploy.sh @@ -0,0 +1,43 @@ +#Arguments: +#$1 - delete stack formation to ensure that all test have same conditon and favour similar ammount of cold start events + +STACK_NAME=dotnet8-native-minimal-api +DELETE_STACK=yes + +COLOR='\033[0;33m' +NO_COLOR='\033[0m' # No Color + +if [ "x$1" != x ]; +then + DELETE_STACK=$1 +fi + +echo "${COLOR}" +echo -------------------------------------------- +echo DELETE_STACK: $DELETE_STACK +echo -------------------------------------------- +echo "${NO_COLOR}" + +if [ $DELETE_STACK == "yes" ]; +then + echo "${COLOR}" + echo -------------------------------------------- + echo DELETING STACK $STACK_NAME + echo -------------------------------------------- + echo "${NO_COLOR}" + aws cloudformation delete-stack --stack-name $STACK_NAME + echo "${COLOR}" + echo --------------------------------------------- + echo Waiting stack to be deleted + echo -------------------------------------------- + echo "${NO_COLOR}" + aws cloudformation wait stack-delete-complete --stack-name $STACK_NAME + echo "${COLOR}" + echo --------------------------------------------- + echo Stack deleted + echo -------------------------------------------- + echo "${NO_COLOR}" +fi + +sam build --use-container --build-image plantpowerjames/dotnet-8-lambda-build:rc2 +sam deploy --stack-name $STACK_NAME --resolve-s3 --s3-prefix $STACK_NAME --no-confirm-changeset --no-fail-on-empty-changeset --capabilities CAPABILITY_IAM \ No newline at end of file diff --git a/src/NET8NativeMinimalAPI/run-loadtest.sh b/src/NET8NativeMinimalAPI/run-loadtest.sh new file mode 100755 index 0000000..c4cd293 --- /dev/null +++ b/src/NET8NativeMinimalAPI/run-loadtest.sh @@ -0,0 +1,149 @@ +#Arguments: +#$1 - load test duration in seconds +#$2 - log interval to be used in the cloudwatch query in minutes +#$3 - when equal to yes cloudwatch log group will be deleted to ensure that only logs of the load test will be evaluated for stat +#$4 - ARN of sns topic to notify test results + +STACK_NAME=dotnet8-native-minimal-api +TEST_DURATIOMN_SEC=60 +LOG_INTERVAL_MIN=20 +LOG_DELETE=yes + +COLOR='\033[0;33m' +NO_COLOR='\033[0m' # No Color + +if [ "x$1" != x ]; +then + TEST_DURATIOMN_SEC=$1 +fi + +if [ "x$2" != x ]; +then + LOG_INTERVAL_MIN=$2 +fi + +if [ "x$3" != x ]; +then + LOG_DELETE=$3 +fi + +if [ "x$4" != x ]; +then + SNS_TOPIC_ARN=$4 +fi + +echo "${COLOR}" +echo -------------------------------------------- +echo DURATION:$TEST_DURATIOMN_SEC +echo LOG INTERVAL:$LOG_INTERVAL_MIN +echo LOG_DELETE: $LOG_DELETE +echo SNS_TOPIC_ARN: $SNS_TOPIC_ARN +echo -------------------------------------------- +echo "${NO_COLOR}" + +mkdir -p Report + +function RunLoadTest() +{ + #Params: + #$1 - Architecture (x86 or arm64).Used for logging and naming report file + #$2 - Stack output name to get API Url + #$3 - Stack output name to get lambda name + + #get test params from cloud formation output + echo "${COLOR}" + export API_URL=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$2'].OutputValue" \ + --output text) + echo API URL: $API_URL + + LAMBDA=$(aws cloudformation describe-stacks --stack-name $STACK_NAME \ + --query "Stacks[0].Outputs[?OutputKey=='$3'].OutputValue" \ + --output text) + echo LAMBDA: $LAMBDA + + if [ $LOG_DELETE == "yes" ]; + then + echo -------------------------------------------- + echo DELETING CLOUDWATCH LOG GROUP /aws/lambda/$LAMBDA + echo -------------------------------------------- + aws logs delete-log-group --log-group-name /aws/lambda/$LAMBDA + echo --------------------------------------------- + echo Waiting 10 sec. for deletion to complete + echo -------------------------------------------- + sleep 10 + fi + + #run load test with artillery + echo -------------------------------------------- + echo $1 RUNNING LOAD TEST $TEST_DURATIOMN_SEC sec $LAMBDA: $API_URL + echo -------------------------------------------- + echo "${NO_COLOR}" + artillery run \ + --overrides '{"config": { "phases": [{ "duration": '$TEST_DURATIOMN_SEC', "arrivalRate": 100 }] } }' \ + --quiet \ + ../../loadtest/load-test.yml + + echo "${COLOR}" + echo -------------------------------------------- + echo Waiting 10 sec. for logs to consolidate + echo -------------------------------------------- + sleep 10 + + #get stats from cloudwatch + enddate=$(date "+%s") + startdate=$(($enddate-($LOG_INTERVAL_MIN*60))) + echo -------------------------------------------- + echo Log start:$startdate end:$enddate + echo -------------------------------------------- + + echo -------------------------------------------- + echo GET ERROR METRICS $1 + echo -------------------------------------------- + aws cloudwatch get-metric-statistics \ + --namespace AWS/Lambda \ + --metric-name Errors \ + --dimensions Name=FunctionName,Value=$LAMBDA \ + --statistics Sum --period 43200 \ + --start-time $startdate --end-time $enddate > ./Report/load-test-errors-$1.json + + QUERY_ID=$(aws logs start-query \ + --log-group-name /aws/lambda/$LAMBDA \ + --start-time $startdate \ + --end-time $enddate \ + --query-string 'filter @type="REPORT" | fields greatest(@initDuration, 0) + @duration as duration, ispresent(@initDuration) as coldstart | stats count(*) as count, pct(duration, 50) as p50, pct(duration, 90) as p90, pct(duration, 99) as p99, max(duration) as max by coldstart' \ + | jq -r '.queryId') + + echo -------------------------------------------- + echo Query started, id: $QUERY_ID + echo -------------------------------------------- + + echo --------------------------------------------- + echo Waiting 10 sec. for cloudwatch query to complete + echo -------------------------------------------- + sleep 10 + + echo -------------------------------------------- + echo RESULTS $LAMBDA + echo -------------------------------------------- + echo "${NO_COLOR}" + date > ./Report/load-test-report-$1.txt + echo $1 RESULTS lambda: $LAMBDA >> ./Report/load-test-report-$1.txt + echo Test duration sec: $TEST_DURATIOMN_SEC >> ./Report/load-test-report-$1.txt + echo Log interval min: $LOG_INTERVAL_MIN >> ./Report/load-test-report-$1.txt + aws logs get-query-results --query-id $QUERY_ID --output text >> ./Report/load-test-report-$1.txt + cat ./Report/load-test-report-$1.txt + aws logs get-query-results --query-id $QUERY_ID --output json >> ./Report/load-test-report-$1.json + + if [ "x$SNS_TOPIC_ARN" != x ]; + then + echo -------------------------------------------- + echo Sending message to sns topic: $SNS_TOPIC_ARN + echo -------------------------------------------- + msg=$(<./Report/load-test-report-$1.txt)\n\n$(<./Report/load-test-errors-$1.json) + subject="serverless dotnet demo load test result for $LAMBDA" + aws sns publish --topic-arn $SNS_TOPIC_ARN --subject "$subject" --message "$msg" + fi +} + +RunLoadTest x86 ApiUrlX86 LambdaX86Name \ No newline at end of file diff --git a/src/NET8NativeMinimalAPI/template.yaml b/src/NET8NativeMinimalAPI/template.yaml index db2eb8e..181d59b 100644 --- a/src/NET8NativeMinimalAPI/template.yaml +++ b/src/NET8NativeMinimalAPI/template.yaml @@ -1,24 +1,28 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 +Parameters: + x86FunctionName: + Type: String + Default: Net8-Native-MinimalApi-X86 + Globals: Function: MemorySize: 1024 - Architectures: ["x86_64"] Runtime: provided.al2 Timeout: 30 Tracing: Active Environment: Variables: PRODUCT_TABLE_NAME: !Ref Table - LOG_GROUP_PREFIX: !Sub "/aws/lambda/net-8-aot-minimal-" - LOAD_TEST_TYPE: "NET 8 native AOT Minimal API" Resources: - ApiFunction: + MinimalApiX86: Type: AWS::Serverless::Function Properties: - CodeUri: . + FunctionName: !Ref x86FunctionName + Architectures: [x86_64] + CodeUri: ./ Handler: ApiBootstrap Events: Api: @@ -30,18 +34,6 @@ Resources: - DynamoDBCrudPolicy: TableName: !Ref Table - - Version: "2012-10-17" - Statement: - - Sid: AllowStartQueries - Effect: Allow - Action: - - logs:DescribeLogGroups - - logs:StartQuery - Resource: "*" - - Sid: AllowGetQueryResults - Effect: Allow - Action: logs:GetQueryResults - Resource: "*" Metadata: BuildMethod: makefile @@ -55,8 +47,16 @@ Resources: KeySchema: - AttributeName: id KeyType: HASH + StreamSpecification: + StreamViewType: NEW_AND_OLD_IMAGES Outputs: ApiUrl: Description: "API Gateway endpoint URL" - Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com/" \ No newline at end of file + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com" + LambdaX86Name: + Description: "Lambda X86 Name" + Value: !Ref x86FunctionName + ApiUrlX86: + Description: "X86 API endpoint URL" + Value: !Sub "https://${ServerlessHttpApi}.execute-api.${AWS::Region}.amazonaws.com" \ No newline at end of file