Skip to content

Commit b07793a

Browse files
sunmou99sunmou99
authored
Setup Desktop test workflow that detects C++ SDK breakage caused by iOS SDK changes (#1162)
1 parent 3095517 commit b07793a

File tree

3 files changed

+66
-20
lines changed

3 files changed

+66
-20
lines changed

‎.github/workflows/integration_tests.yml

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ name: Integration tests
33
on:
44
schedule:
55
- cron: "0 9 * * *" # 9am UTC = 1am PST / 2am PDT. for all testapps except firestore
6-
- cron: "0 10 * * *" # 10am UTC = 2am PST / 3am PDT. for firestore test only
6+
- cron: "0 10 * * *" # 10am UTC = 2am PST / 3am PDT. for firestore test against release iOS/Android SDK
7+
- cron: "0 11 * * *" # 11am UTC = 3am PST / 4am PDT. for firestore desktop test aginst tip-of-tree iOS repo
78

89
pull_request:
910
types: [ labeled, closed ]
@@ -183,17 +184,25 @@ jobs:
183184
# To feed input into the job matrix, we first need to convert to a JSON
184185
# list. Then we can use fromJson to define the field in the matrix for the tests job.
185186
if [[ "${{ github.event.schedule }}" == "0 9 * * *" ]]; then
186-
# at 1am PST / 2am PDT. Running integration tests and generate test report for all testapps except firestore
187+
# at 1am PST/2am PDT. Running integration tests and generate test report for all testapps except firestore
187188
apis="admob,analytics,auth,database,dynamic_links,functions,gma,installations,messaging,remote_config,storage"
188-
elif [[ "${{ github.event.schedule }}" == "0 10 * * *" ]]; then
189-
# at 2am PST / 3am PDT. Running integration tests for firestore and generate test report
189+
elif [[ "${{ github.event.schedule }}" == "0 10 * * *" || "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
190+
# at 2am PST/3am PDT and 3am PST/4am PDT. Running integration tests for firestore and generate test report.
190191
apis="firestore"
191192
else
192193
apis=$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k apis -o "${{github.event.inputs.apis}}" ${TEST_MATRIX_PARAM} )
193194
fi
195+
if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
196+
# at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
197+
matrix_platform="Desktop"
198+
matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "ubuntu-20.04,macos-12")
199+
else
200+
matrix_platform=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )
201+
matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")
202+
fi
194203
echo "apis=${apis}" >> $GITHUB_OUTPUT
195-
echo "matrix_platform=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k platform -o "${{github.event.inputs.platforms}}" --apis ${apis} )" >> $GITHUB_OUTPUT
196-
echo "matrix_os=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k os -o "${{github.event.inputs.operating_systems}}")" >> $GITHUB_OUTPUT
204+
echo "matrix_platform=${matrix_platform}" >> $GITHUB_OUTPUT
205+
echo "matrix_os=${matrix_os}" >> $GITHUB_OUTPUT
197206
echo "matrix_arch_macos=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_macos)" >> $GITHUB_OUTPUT
198207
echo "matrix_arch_windows_linux=$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k architecture_windows_linux)" >> $GITHUB_OUTPUT
199208
# Combine architecture_macos and architecture_windows_linux to get a list of all architectures for the build matrix.
@@ -318,6 +327,10 @@ jobs:
318327
additional_flags+=(--cmake_flag=-DFIREBASE_USE_BORINGSSL=ON)
319328
fi
320329
fi
330+
if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
331+
# at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
332+
additional_flags+=(--cmake_flag=-DFIRESTORE_DEP_SOURCE=TIP)
333+
fi
321334
python scripts/gha/build_testapps.py --p Desktop \
322335
--t ${{ needs.check_and_prepare.outputs.apis }} \
323336
--output_directory "${{ github.workspace }}" \
@@ -327,6 +340,13 @@ jobs:
327340
--gha_build \
328341
--arch ${{ matrix.arch }} \
329342
${additional_flags[*]}
343+
- name: Upload Desktop Cmake
344+
uses: actions/upload-artifact@v3
345+
if: ${{ !cancelled() }}
346+
with:
347+
name: cmake-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.ssl_variant }}
348+
path: D:/a/firebase-cpp-sdk/firebase-cpp-sdk/ta/firestore/iti/CMakeFiles/
349+
retention-days: 1
330350
- name: Prepare results summary artifact
331351
if: ${{ !cancelled() }}
332352
shell: bash
@@ -1361,7 +1381,12 @@ jobs:
13611381
if [[ "${{ github.event.inputs.test_pull_request }}" == "nightly-packaging" ]]; then
13621382
additional_flags=(--build_against sdk)
13631383
else
1364-
additional_flags=(--build_against repo)
1384+
if [[ "${{ github.event.schedule }}" == "0 11 * * *" ]]; then
1385+
# at 3am PST/4am PDT. Running firestore desktop integration test aginst tip-of-tree ios repo
1386+
additional_flags=(--build_against tip)
1387+
else
1388+
additional_flags=(--build_against repo)
1389+
fi
13651390
fi
13661391
if [[ "${{ needs.check_and_prepare.outputs.apis }}" == "firestore" ]]; then
13671392
additional_flags+=(--build_apis firestore)

‎scripts/gha/build_desktop.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ def cmake_configure(build_dir, arch, msvc_runtime_library='static', linux_abi='l
230230
if verbose:
231231
cmd.append('-DCMAKE_VERBOSE_MAKEFILE=1')
232232

233-
utils.run_command(cmd)
233+
utils.run_command(cmd, check=True)
234234

235235
def main():
236236
args = parse_cmdline_args()
@@ -269,7 +269,7 @@ def main():
269269
# Example: cmake --build build -j 8 --target firebase_app firebase_auth
270270
cmd.append('--target')
271271
cmd.extend(args.target)
272-
utils.run_command(cmd)
272+
utils.run_command(cmd, check=True)
273273

274274

275275
def parse_cmdline_args():

‎scripts/gha/it_workflow.py

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -66,17 +66,20 @@
6666
_COMMENT_TITLE_FLAKY = "### Integration test with FLAKINESS (succeeded after retry)\n"
6767
_COMMENT_TITLE_FAIL = "### ❌  Integration test FAILED\n"
6868
_COMMENT_TITLE_SUCCEED = "### ✅  Integration test succeeded!\n"
69-
_COMMENT_TITLE_FLAKY_SDK = "\n***\n### [build against SDK] Integration test with FLAKINESS (succeeded after retry)\n"
70-
_COMMENT_TITLE_FAIL_SDK = "\n***\n### ❌  [build against SDK] Integration test FAILED\n"
71-
_COMMENT_TITLE_SUCCEED_SDK = "\n***\n### ✅  [build against SDK] Integration test succeeded!\n"
7269
_COMMENT_TITLE_FLAKY_REPO = "### [build against repo] Integration test with FLAKINESS (succeeded after retry)\n"
7370
_COMMENT_TITLE_FAIL_REPO = "### ❌  [build against repo] Integration test FAILED\n"
7471
_COMMENT_TITLE_SUCCEED_REPO = "### ✅  [build against repo] Integration test succeeded!\n"
72+
_COMMENT_TITLE_FLAKY_SDK = "\n***\n### [build against SDK] Integration test with FLAKINESS (succeeded after retry)\n"
73+
_COMMENT_TITLE_FAIL_SDK = "\n***\n### ❌  [build against SDK] Integration test FAILED\n"
74+
_COMMENT_TITLE_SUCCEED_SDK = "\n***\n### ✅  [build against SDK] Integration test succeeded!\n"
75+
_COMMENT_TITLE_FLAKY_TIP = "\n***\n### [build against tip] Integration test with FLAKINESS (succeeded after retry)\n"
76+
_COMMENT_TITLE_FAIL_TIP = "\n***\n### ❌  [build against tip] Integration test FAILED\n"
77+
_COMMENT_TITLE_SUCCEED_TIP = "\n***\n### ✅  [build against tip] Integration test succeeded!\n"
7578

7679
_COMMENT_FLAKY_TRACKER = "\n\nAdd flaky tests to **[go/fpl-cpp-flake-tracker](http://go/fpl-cpp-flake-tracker)**\n"
7780

7881
_COMMENT_IDENTIFIER = "integration-test-status-comment"
79-
_COMMENT_HIDDEN_DIVIDER = f'\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>\n'
82+
_COMMENT_HIDDEN_DIVIDER = f'\r\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>\r\n'
8083

8184
_LOG_ARTIFACT_NAME = "log-artifact"
8285
_LOG_OUTPUT_DIR = "test_results"
@@ -89,6 +92,7 @@
8992

9093
_BUILD_AGAINST_SDK = "sdk"
9194
_BUILD_AGAINST_REPO = "repo"
95+
_BUILD_AGAINST_TIP = "tip"
9296

9397
_BUILD_API_ALL = "all"
9498
_BUILD_API_FIRESTORE = "firestore"
@@ -222,27 +226,44 @@ def test_report(token, actor, commit, run_id, build_against, build_apis):
222226

223227
issue_number = _get_issue_number(token, report_title, _REPORT_LABEL)
224228
previous_comment = github.get_issue_body(token, issue_number)
225-
[_, previous_comment_repo, previous_comment_sdk] = previous_comment.split(_COMMENT_HIDDEN_DIVIDER)
229+
[_, previous_comment_repo, previous_comment_sdk, previous_comment_tip] = previous_comment.split(_COMMENT_HIDDEN_DIVIDER)
226230
success_or_only_flakiness, log_summary = _get_summary_table(token, run_id)
227231
if success_or_only_flakiness and not log_summary:
228232
# succeeded (without flakiness)
229-
title = _COMMENT_TITLE_SUCCEED_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_SUCCEED_SDK
233+
if build_against==_BUILD_AGAINST_REPO:
234+
title = _COMMENT_TITLE_SUCCEED_REPO
235+
elif build_against==_BUILD_AGAINST_SDK:
236+
title = _COMMENT_TITLE_SUCCEED_SDK
237+
else:
238+
title = _COMMENT_TITLE_SUCCEED_TIP
230239
comment = title + _get_description(actor, commit, run_id)
231240
else:
232241
if success_or_only_flakiness:
233242
# all failures/errors are due to flakiness (succeeded after retry)
234-
title = _COMMENT_TITLE_FLAKY_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_FLAKY_SDK
243+
if build_against==_BUILD_AGAINST_REPO:
244+
title = _COMMENT_TITLE_FLAKY_REPO
245+
elif build_against==_BUILD_AGAINST_SDK:
246+
title = _COMMENT_TITLE_FLAKY_SDK
247+
else:
248+
title = _COMMENT_TITLE_FLAKY_TIP
235249
else:
236250
# failures/errors still exist after retry
237-
title = _COMMENT_TITLE_FAIL_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_FAIL_SDK
251+
if build_against==_BUILD_AGAINST_REPO:
252+
title = _COMMENT_TITLE_FAIL_REPO
253+
elif build_against==_BUILD_AGAINST_SDK:
254+
title = _COMMENT_TITLE_FAIL_SDK
255+
else:
256+
title = _COMMENT_TITLE_FAIL_TIP
238257
comment = title + _get_description(actor, commit, run_id) + log_summary + _COMMENT_FLAKY_TRACKER
239258

240259
if build_against==_BUILD_AGAINST_REPO:
241-
comment = prefix + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk
260+
comment = prefix + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk + _COMMENT_HIDDEN_DIVIDER + previous_comment_tip
242261
elif build_against==_BUILD_AGAINST_SDK:
243-
comment = prefix + _COMMENT_HIDDEN_DIVIDER + previous_comment_repo + _COMMENT_HIDDEN_DIVIDER + comment
262+
comment = prefix + _COMMENT_HIDDEN_DIVIDER + previous_comment_repo + _COMMENT_HIDDEN_DIVIDER + comment + _COMMENT_HIDDEN_DIVIDER + previous_comment_tip
263+
else:
264+
comment = prefix + _COMMENT_HIDDEN_DIVIDER + previous_comment_repo + _COMMENT_HIDDEN_DIVIDER + previous_comment_sdk + _COMMENT_HIDDEN_DIVIDER + comment
244265

245-
if (_COMMENT_TITLE_SUCCEED_REPO in comment) and (_COMMENT_TITLE_SUCCEED_SDK in comment):
266+
if (_COMMENT_TITLE_SUCCEED_REPO in comment) and (_COMMENT_TITLE_SUCCEED_SDK in comment) and (build_apis != _BUILD_API_FIRESTORE or _COMMENT_TITLE_SUCCEED_TIP in comment):
246267
github.close_issue(token, issue_number)
247268
else:
248269
github.open_issue(token, issue_number)

0 commit comments

Comments
 (0)