From 4837b47da785827307116ae1b1b036898ce4166d Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Wed, 26 Feb 2025 16:41:07 +0000 Subject: [PATCH 01/28] Delete files which were removed in source dir Using 'az storage blob sync' to sync the content of the dir --- .github/workflows/docs-build-push.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index add7934..04b1e46 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -255,13 +255,12 @@ jobs: with: inlineScript: | cd ${{inputs.docs_build_path}} \ - && az storage blob upload-batch \ + && az storage blob sync \ -s $DOCS_SOURCE_PATH \ - -d '$web' \ - --destination-path "dev/${{github.repository}}/previews/${PR_NUMBER}" \ + -c '$web' \ + -d "dev/${{github.repository}}/previews/${PR_NUMBER}" \ --account-name ${{steps.keyvault.outputs.accountName}} \ - --overwrite \ - --content-cache-control "max-age=3600" \ + --delete-destination true \ --auth-mode login az afd endpoint purge \ From 9f7c0c395420f9457c8554132d1eb23358781e50 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 13:33:42 +0000 Subject: [PATCH 02/28] Update docs-build-push.yml AZCOPY_AUTO_LOGIN_TYPE --- .github/workflows/docs-build-push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 04b1e46..11bdc73 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -254,14 +254,14 @@ jobs: if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | + export AZCOPY_AUTO_LOGIN_TYPE=AZCLI cd ${{inputs.docs_build_path}} \ && az storage blob sync \ -s $DOCS_SOURCE_PATH \ -c '$web' \ -d "dev/${{github.repository}}/previews/${PR_NUMBER}" \ --account-name ${{steps.keyvault.outputs.accountName}} \ - --delete-destination true \ - --auth-mode login + --delete-destination true az afd endpoint purge \ --resource-group ${{steps.keyvault.outputs.resourceGroupName}} \ From b948bc5fd4dc8f19e8710edc73335be9caaad003 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 13:54:26 +0000 Subject: [PATCH 03/28] Tenant ID --- .github/workflows/docs-build-push.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 11bdc73..761d132 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -7,6 +7,8 @@ on: required: true AZURE_KEY_VAULT: required: true + TN: + required: true inputs: environment: description: "This will be appended to the baseURL for for production builds. For example, main docs will be `/` where agent would be `/nginx-agent`" @@ -255,6 +257,7 @@ jobs: with: inlineScript: | export AZCOPY_AUTO_LOGIN_TYPE=AZCLI + export AZCOPY_TENANT_ID=${{ secrets.TN }} cd ${{inputs.docs_build_path}} \ && az storage blob sync \ -s $DOCS_SOURCE_PATH \ From 686b4450ad037670f03d100c7363042028433474 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 13:56:55 +0000 Subject: [PATCH 04/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 761d132..82c5265 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -7,8 +7,6 @@ on: required: true AZURE_KEY_VAULT: required: true - TN: - required: true inputs: environment: description: "This will be appended to the baseURL for for production builds. For example, main docs will be `/` where agent would be `/nginx-agent`" From 760064319ed02c6a069bd2cca7f915ab7244cef7 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 14:02:41 +0000 Subject: [PATCH 05/28] env --- .github/workflows/docs-build-push.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 82c5265..20439c1 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -92,7 +92,9 @@ jobs: THEME_VERSION: ${{inputs.force_hugo_theme_version}} AUTO_DEPLOY_BRANCH: ${{inputs.auto_deploy_branch}} AUTO_DEPLOY_ENV: ${{inputs.auto_deploy_env}} - + AZCOPY_AUTO_LOGIN_TYPE: AZCLI + AZCOPY_TENANT_ID: ${{ secrets.TN }} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} steps: @@ -254,8 +256,6 @@ jobs: if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | - export AZCOPY_AUTO_LOGIN_TYPE=AZCLI - export AZCOPY_TENANT_ID=${{ secrets.TN }} cd ${{inputs.docs_build_path}} \ && az storage blob sync \ -s $DOCS_SOURCE_PATH \ From 0b58ced9a28fbc533c1fd1c3864f3645279d0665 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 14:48:25 +0000 Subject: [PATCH 06/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 20439c1..c619730 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -7,6 +7,8 @@ on: required: true AZURE_KEY_VAULT: required: true + AZURE_TENANT_ID: + required: true inputs: environment: description: "This will be appended to the baseURL for for production builds. For example, main docs will be `/` where agent would be `/nginx-agent`" @@ -93,7 +95,7 @@ jobs: AUTO_DEPLOY_BRANCH: ${{inputs.auto_deploy_branch}} AUTO_DEPLOY_ENV: ${{inputs.auto_deploy_env}} AZCOPY_AUTO_LOGIN_TYPE: AZCLI - AZCOPY_TENANT_ID: ${{ secrets.TN }} + AZCOPY_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 620f87af2ccfb82e796b8ce48a6d11c254496194 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 15:05:30 +0000 Subject: [PATCH 07/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index c619730..f1f3c44 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -7,8 +7,6 @@ on: required: true AZURE_KEY_VAULT: required: true - AZURE_TENANT_ID: - required: true inputs: environment: description: "This will be appended to the baseURL for for production builds. For example, main docs will be `/` where agent would be `/nginx-agent`" @@ -94,8 +92,6 @@ jobs: THEME_VERSION: ${{inputs.force_hugo_theme_version}} AUTO_DEPLOY_BRANCH: ${{inputs.auto_deploy_branch}} AUTO_DEPLOY_ENV: ${{inputs.auto_deploy_env}} - AZCOPY_AUTO_LOGIN_TYPE: AZCLI - AZCOPY_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -253,6 +249,12 @@ jobs: ### Azure upload + - name: AzCopy env vars + run: | + echo "AZCOPY_TENANT_ID=`echo ${{secrets.AZURE_CREDENTIALS}} | jq -r .tenantId` >> $GITHUB_ENV + echo "AZCOPY_AUTO_LOGIN_TYPE=AZCLI" >> $GITHUB_ENV + + - name: Azure upload PR preview uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' From aa7e4f119104ebcd286e0be8ba3a829b9645aefa Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 15:11:08 +0000 Subject: [PATCH 08/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index f1f3c44..69eb84d 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -251,7 +251,7 @@ jobs: - name: AzCopy env vars run: | - echo "AZCOPY_TENANT_ID=`echo ${{secrets.AZURE_CREDENTIALS}} | jq -r .tenantId` >> $GITHUB_ENV + echo "AZCOPY_TENANT_ID=`echo ${{secrets.AZURE_CREDENTIALS}} | jq -r .tenantId`" >> $GITHUB_ENV echo "AZCOPY_AUTO_LOGIN_TYPE=AZCLI" >> $GITHUB_ENV From 0a06bbdc4bc8213d67f810ddeaf4640551cecc7f Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 15:46:33 +0000 Subject: [PATCH 09/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 69eb84d..7fdae47 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -250,10 +250,13 @@ jobs: ### Azure upload - name: AzCopy env vars - run: | - echo "AZCOPY_TENANT_ID=`echo ${{secrets.AZURE_CREDENTIALS}} | jq -r .tenantId`" >> $GITHUB_ENV - echo "AZCOPY_AUTO_LOGIN_TYPE=AZCLI" >> $GITHUB_ENV - + uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 + id: key + with: + inlineScript: | + key=$(az storage account keys list --resource-group ${{steps.keyvault.outputs.resourceGroupName}} --account-name ${{steps.keyvault.outputs.accountName}} --query "[0].value" --output tsv) + echo "::add-mask::$key" + echo "storageAccountKey=$key" >> $GITHUB_OUTPUT - name: Azure upload PR preview uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 @@ -265,7 +268,9 @@ jobs: -s $DOCS_SOURCE_PATH \ -c '$web' \ -d "dev/${{github.repository}}/previews/${PR_NUMBER}" \ + --auth-mode key \ --account-name ${{steps.keyvault.outputs.accountName}} \ + --account-key ${{steps.id.outputs.storageAccountKey}} \ --delete-destination true az afd endpoint purge \ From a3f03bacead5ed18b50b3edf31d56f4e519ffbc2 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 16:07:09 +0000 Subject: [PATCH 10/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 7fdae47..741fd22 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -171,6 +171,17 @@ jobs: echo "$secret_get=$value" >> $GITHUB_OUTPUT done + - name: az key + id: key + uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 + with: + inlineScript: | + az account show + az storage -h + key=$(az storage account keys list --resource-group ${{steps.keyvault.outputs.resourceGroupName}} --account-name ${{steps.keyvault.outputs.accountName}} --query "[0].value" --output tsv) + echo "::add-mask::$key" + echo "storageAccountKey=$key" >> $GITHUB_OUTPUT + - name: Checkout docs content uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.7.1 with: @@ -248,15 +259,6 @@ jobs: make deploy ### Azure upload - - - name: AzCopy env vars - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - id: key - with: - inlineScript: | - key=$(az storage account keys list --resource-group ${{steps.keyvault.outputs.resourceGroupName}} --account-name ${{steps.keyvault.outputs.accountName}} --query "[0].value" --output tsv) - echo "::add-mask::$key" - echo "storageAccountKey=$key" >> $GITHUB_OUTPUT - name: Azure upload PR preview uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 From d14f145801f46c52193c9f7c8ff0c2000795a980 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 16:19:10 +0000 Subject: [PATCH 11/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 741fd22..1f52937 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -178,9 +178,10 @@ jobs: inlineScript: | az account show az storage -h - key=$(az storage account keys list --resource-group ${{steps.keyvault.outputs.resourceGroupName}} --account-name ${{steps.keyvault.outputs.accountName}} --query "[0].value" --output tsv) + key=$(az storage account show-connection-string --name ${{steps.keyvault.outputs.accountName}} --resource-group ${{steps.keyvault.outputs.resourceGroupName}}) + #key=$(az storage account keys list --resource-group ${{steps.keyvault.outputs.resourceGroupName}} --account-name ${{steps.keyvault.outputs.accountName}} --query "[0].value" --output tsv) echo "::add-mask::$key" - echo "storageAccountKey=$key" >> $GITHUB_OUTPUT + echo "sharedAccessSignature=$key" >> $GITHUB_OUTPUT - name: Checkout docs content uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.7.1 From 67e6510647d953d6de328d8eb19a0b9b88af390d Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 16:47:34 +0000 Subject: [PATCH 12/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 1f52937..da9ef2c 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -177,12 +177,8 @@ jobs: with: inlineScript: | az account show - az storage -h - key=$(az storage account show-connection-string --name ${{steps.keyvault.outputs.accountName}} --resource-group ${{steps.keyvault.outputs.resourceGroupName}}) - #key=$(az storage account keys list --resource-group ${{steps.keyvault.outputs.resourceGroupName}} --account-name ${{steps.keyvault.outputs.accountName}} --query "[0].value" --output tsv) - echo "::add-mask::$key" - echo "sharedAccessSignature=$key" >> $GITHUB_OUTPUT - + azcopy login + - name: Checkout docs content uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.7.1 with: @@ -271,9 +267,8 @@ jobs: -s $DOCS_SOURCE_PATH \ -c '$web' \ -d "dev/${{github.repository}}/previews/${PR_NUMBER}" \ - --auth-mode key \ + --auth-mode login \ --account-name ${{steps.keyvault.outputs.accountName}} \ - --account-key ${{steps.id.outputs.storageAccountKey}} \ --delete-destination true az afd endpoint purge \ From fac110e07410f9f06e94649670146f3a081aee14 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 16:56:00 +0000 Subject: [PATCH 13/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index da9ef2c..fe6545c 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -170,14 +170,6 @@ jobs: echo "::add-mask::$value" echo "$secret_get=$value" >> $GITHUB_OUTPUT done - - - name: az key - id: key - uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 - with: - inlineScript: | - az account show - azcopy login - name: Checkout docs content uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.7.1 @@ -263,7 +255,7 @@ jobs: with: inlineScript: | cd ${{inputs.docs_build_path}} \ - && az storage blob sync \ + && az storage azcopy blob sync \ -s $DOCS_SOURCE_PATH \ -c '$web' \ -d "dev/${{github.repository}}/previews/${PR_NUMBER}" \ From 5b80d75a6a9f9a003ba117613eff81a2ae5b3373 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 17:00:02 +0000 Subject: [PATCH 14/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index fe6545c..ed552ec 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -254,14 +254,14 @@ jobs: if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | + az config set extension.dynamic_install_allow_preview=true cd ${{inputs.docs_build_path}} \ && az storage azcopy blob sync \ - -s $DOCS_SOURCE_PATH \ - -c '$web' \ - -d "dev/${{github.repository}}/previews/${PR_NUMBER}" \ + --source $DOCS_SOURCE_PATH \ + --container '$web' \ + --destination "dev/${{github.repository}}/previews/${PR_NUMBER}" \ --auth-mode login \ - --account-name ${{steps.keyvault.outputs.accountName}} \ - --delete-destination true + --account-name ${{steps.keyvault.outputs.accountName}} az afd endpoint purge \ --resource-group ${{steps.keyvault.outputs.resourceGroupName}} \ From 12d0f0bd7dab3c643b2ad343db0a2e740809c615 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 17:07:01 +0000 Subject: [PATCH 15/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index ed552ec..b1b7e9a 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -254,13 +254,13 @@ jobs: if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | - az config set extension.dynamic_install_allow_preview=true + cd ${{inputs.docs_build_path}} \ - && az storage azcopy blob sync \ + + && az storage blob sync \ --source $DOCS_SOURCE_PATH \ --container '$web' \ --destination "dev/${{github.repository}}/previews/${PR_NUMBER}" \ - --auth-mode login \ --account-name ${{steps.keyvault.outputs.accountName}} az afd endpoint purge \ From bed07dcdb6ad2e6f8197bc45f2239b469db2b722 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 27 Feb 2025 17:09:15 +0000 Subject: [PATCH 16/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index b1b7e9a..ce3205c 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -254,9 +254,7 @@ jobs: if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | - cd ${{inputs.docs_build_path}} \ - && az storage blob sync \ --source $DOCS_SOURCE_PATH \ --container '$web' \ From 5d06e224eb4d0e76c3776a05c62ba1e929ccbe08 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Wed, 12 Mar 2025 17:12:35 +0000 Subject: [PATCH 17/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 33 ++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index ce3205c..e2eb13f 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -254,12 +254,33 @@ jobs: if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | - cd ${{inputs.docs_build_path}} \ - && az storage blob sync \ - --source $DOCS_SOURCE_PATH \ - --container '$web' \ - --destination "dev/${{github.repository}}/previews/${PR_NUMBER}" \ - --account-name ${{steps.keyvault.outputs.accountName}} + cd ${{inputs.docs_build_path}} \ + && az storage blob upload-batch \ + -s $DOCS_SOURCE_PATH \ + -d '$web' \ + --destination-path "dev/${{github.repository}}/previews/${PR_NUMBER}-new" \ + --account-name ${{steps.keyvault.outputs.accountName}} \ + --overwrite \ + --content-cache-control "max-age=3600" \ + --auth-mode login + && az storage fs directory move \ + --account-name ${{steps.keyvault.outputs.accountName}} \ + --auth-mode login \ + --file-system '$web' \ + --name "dev/${{github.repository}}/previews/${PR_NUMBER}" \ + --new-directory "dev/${{github.repository}}/previews/${PR_NUMBER}-old" \ + && az storage fs directory move \ + --account-name ${{steps.keyvault.outputs.accountName}} \ + --auth-mode login \ + --file-system '$web' \ + --name "dev/${{github.repository}}/previews/${PR_NUMBER}-new" \ + --new-directory "dev/${{github.repository}}/previews/${PR_NUMBER}" \ + && az storage fs directory delete \ + --account-name ${{steps.keyvault.outputs.accountName}} \ + --auth-mode login \ + --file-system '$web' \ + --name "dev/${{github.repository}}/previews/${PR_NUMBER}-old" \ + --yes az afd endpoint purge \ --resource-group ${{steps.keyvault.outputs.resourceGroupName}} \ From 76a2d34ea087bcdee5bacd1ce75bbc1ac416f781 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Wed, 12 Mar 2025 17:17:18 +0000 Subject: [PATCH 18/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index e2eb13f..d5a4a04 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -254,8 +254,8 @@ jobs: if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | - cd ${{inputs.docs_build_path}} \ - && az storage blob upload-batch \ + cd ${{inputs.docs_build_path}} + az storage blob upload-batch \ -s $DOCS_SOURCE_PATH \ -d '$web' \ --destination-path "dev/${{github.repository}}/previews/${PR_NUMBER}-new" \ @@ -263,19 +263,22 @@ jobs: --overwrite \ --content-cache-control "max-age=3600" \ --auth-mode login - && az storage fs directory move \ + + az storage fs directory move \ --account-name ${{steps.keyvault.outputs.accountName}} \ --auth-mode login \ --file-system '$web' \ --name "dev/${{github.repository}}/previews/${PR_NUMBER}" \ - --new-directory "dev/${{github.repository}}/previews/${PR_NUMBER}-old" \ - && az storage fs directory move \ + --new-directory "dev/${{github.repository}}/previews/${PR_NUMBER}-old" + + az storage fs directory move \ --account-name ${{steps.keyvault.outputs.accountName}} \ --auth-mode login \ --file-system '$web' \ --name "dev/${{github.repository}}/previews/${PR_NUMBER}-new" \ - --new-directory "dev/${{github.repository}}/previews/${PR_NUMBER}" \ - && az storage fs directory delete \ + --new-directory "dev/${{github.repository}}/previews/${PR_NUMBER}" + + az storage fs directory delete \ --account-name ${{steps.keyvault.outputs.accountName}} \ --auth-mode login \ --file-system '$web' \ From 2718fe28b1c16abb7eb48293c68894abb22a41db Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Wed, 12 Mar 2025 18:09:57 +0000 Subject: [PATCH 19/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 39 +++++++-------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index d5a4a04..4283344 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -248,6 +248,13 @@ jobs: make deploy ### Azure upload + - name: Install azcopy + run: | + wget https://aka.ms/downloadazcopy-v10-linux + tar -xvf downloadazcopy-v10-linux + sudo cp ./azcopy_linux_amd64_*/azcopy . + sudo chmod 755 azcopy + export "AZCOPY_AUTO_LOGIN_TYPE=AZCLI" >> $GITHUB_ENV - name: Azure upload PR preview uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 @@ -255,36 +262,8 @@ jobs: with: inlineScript: | cd ${{inputs.docs_build_path}} - az storage blob upload-batch \ - -s $DOCS_SOURCE_PATH \ - -d '$web' \ - --destination-path "dev/${{github.repository}}/previews/${PR_NUMBER}-new" \ - --account-name ${{steps.keyvault.outputs.accountName}} \ - --overwrite \ - --content-cache-control "max-age=3600" \ - --auth-mode login - - az storage fs directory move \ - --account-name ${{steps.keyvault.outputs.accountName}} \ - --auth-mode login \ - --file-system '$web' \ - --name "dev/${{github.repository}}/previews/${PR_NUMBER}" \ - --new-directory "dev/${{github.repository}}/previews/${PR_NUMBER}-old" - - az storage fs directory move \ - --account-name ${{steps.keyvault.outputs.accountName}} \ - --auth-mode login \ - --file-system '$web' \ - --name "dev/${{github.repository}}/previews/${PR_NUMBER}-new" \ - --new-directory "dev/${{github.repository}}/previews/${PR_NUMBER}" - - az storage fs directory delete \ - --account-name ${{steps.keyvault.outputs.accountName}} \ - --auth-mode login \ - --file-system '$web' \ - --name "dev/${{github.repository}}/previews/${PR_NUMBER}-old" \ - --yes - + ${GITHUB_WORKSPACE}/azcopy sync "$DOCS_SOURCE_PATH" "https://${{steps.keyvault.outputs.accountName}}.blob.core.windows.net/${web}/dev/${{github.repository}}/previews/${PR_NUMBER}" + az afd endpoint purge \ --resource-group ${{steps.keyvault.outputs.resourceGroupName}} \ --profile-name ${{steps.keyvault.outputs.cdnProfileName}} \ From f1d5756b39ccfd69434ff8bd55a0ea9a466eb204 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Wed, 12 Mar 2025 18:21:41 +0000 Subject: [PATCH 20/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 4283344..a513be4 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -254,7 +254,7 @@ jobs: tar -xvf downloadazcopy-v10-linux sudo cp ./azcopy_linux_amd64_*/azcopy . sudo chmod 755 azcopy - export "AZCOPY_AUTO_LOGIN_TYPE=AZCLI" >> $GITHUB_ENV + echo "AZCOPY_AUTO_LOGIN_TYPE=AZCLI" >> $GITHUB_ENV - name: Azure upload PR preview uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 @@ -262,6 +262,8 @@ jobs: with: inlineScript: | cd ${{inputs.docs_build_path}} + env + ${GITHUB_WORKSPACE}/azcopy -v ${GITHUB_WORKSPACE}/azcopy sync "$DOCS_SOURCE_PATH" "https://${{steps.keyvault.outputs.accountName}}.blob.core.windows.net/${web}/dev/${{github.repository}}/previews/${PR_NUMBER}" az afd endpoint purge \ From b9aa10ffd7fa574741b921361616b6e0c0eda0e9 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 13 Mar 2025 12:01:24 +0000 Subject: [PATCH 21/28] Update docs-build-push.yml. List of existing blobs --- .github/workflows/docs-build-push.yml | 44 ++++++++++++++++++--------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index a513be4..40d6912 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -92,7 +92,7 @@ jobs: THEME_VERSION: ${{inputs.force_hugo_theme_version}} AUTO_DEPLOY_BRANCH: ${{inputs.auto_deploy_branch}} AUTO_DEPLOY_ENV: ${{inputs.auto_deploy_env}} - + concurrency: group: ${{ github.workflow }}-${{ github.ref }} steps: @@ -170,7 +170,7 @@ jobs: echo "::add-mask::$value" echo "$secret_get=$value" >> $GITHUB_OUTPUT done - + - name: Checkout docs content uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.7.1 with: @@ -248,24 +248,40 @@ jobs: make deploy ### Azure upload - - name: Install azcopy - run: | - wget https://aka.ms/downloadazcopy-v10-linux - tar -xvf downloadazcopy-v10-linux - sudo cp ./azcopy_linux_amd64_*/azcopy . - sudo chmod 755 azcopy - echo "AZCOPY_AUTO_LOGIN_TYPE=AZCLI" >> $GITHUB_ENV + - name: List of esisting blobs + uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 + if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' + with: + inlineScript: | + az storage blob list \ + --container-name '$web' \ + --account-name ${{steps.keyvault.outputs.accountName}} \ + --auth-mode login \ + --prefix "dev/${{github.repository}}/previews/${PR_NUMBER}" > existing_blobs.txt + echo "Info about existing blobs" + cat existing_blobs.txt | wc -l + tail existing_blobs.txt + echo "Source" + ls $DOCS_SOURCE_PATH + exit 1 + + - name: Azure upload PR preview uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | - cd ${{inputs.docs_build_path}} - env - ${GITHUB_WORKSPACE}/azcopy -v - ${GITHUB_WORKSPACE}/azcopy sync "$DOCS_SOURCE_PATH" "https://${{steps.keyvault.outputs.accountName}}.blob.core.windows.net/${web}/dev/${{github.repository}}/previews/${PR_NUMBER}" - + cd ${{inputs.docs_build_path}} \ + && az storage blob upload-batch \ + -s $DOCS_SOURCE_PATH \ + -d '$web' \ + --destination-path "dev/${{github.repository}}/previews/${PR_NUMBER}" \ + --account-name ${{steps.keyvault.outputs.accountName}} \ + --overwrite \ + --content-cache-control "max-age=3600" \ + --auth-mode login + az afd endpoint purge \ --resource-group ${{steps.keyvault.outputs.resourceGroupName}} \ --profile-name ${{steps.keyvault.outputs.cdnProfileName}} \ From 5c5adf4513aa80f5bec74c6be46046473efaf2d4 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 13 Mar 2025 12:15:09 +0000 Subject: [PATCH 22/28] Update docs-build-push.yml the the full list of blobs --- .github/workflows/docs-build-push.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 40d6912..8965709 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -259,10 +259,7 @@ jobs: --auth-mode login \ --prefix "dev/${{github.repository}}/previews/${PR_NUMBER}" > existing_blobs.txt echo "Info about existing blobs" - cat existing_blobs.txt | wc -l - tail existing_blobs.txt - echo "Source" - ls $DOCS_SOURCE_PATH + cat existing_blobs.txt exit 1 From 17f72007d2d7611a2de66ea2a13b50dfc292bca9 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 13 Mar 2025 14:14:44 +0000 Subject: [PATCH 23/28] Update docs-build-push.yml remove function preview --- .github/workflows/docs-build-push.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 8965709..ccb9c4c 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -253,15 +253,27 @@ jobs: if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | + dstprefix="dev/${{github.repository}}/previews/${PR_NUMBER}" + srcprefix="$DOCS_SOURCE_PATH" az storage blob list \ --container-name '$web' \ --account-name ${{steps.keyvault.outputs.accountName}} \ --auth-mode login \ - --prefix "dev/${{github.repository}}/previews/${PR_NUMBER}" > existing_blobs.txt - echo "Info about existing blobs" - cat existing_blobs.txt - exit 1 - + --prefix "${dstprefix}/" > blobs.txt + cat blobs.txt | grep name | cut -d: -f 2 | cut -d'"' -f2 > blobs_paths.txt + cat blobs_paths.txt | sed -e "s#${dstprefix}/##g" > blobs_names.txt + find ${srcprefix} -type f > local_paths.txt + cat local_paths.txt | sed -e "s#${srcprefix}/##g" > local_names.txt + grep -Fxv -f local_names.txt blobs_names.txt > delete.txt + for file in `cat delete.txt` ; do + # we have to remove the file + # az storage blob delete \ + # --container-name '$web' + # --name ${dstprefix}/${file} + # --account-name ${{steps.keyvault.outputs.accountName}} \ + # --auth-mode login + echo "${dstprefix}/${file} should be removed" + done - name: Azure upload PR preview From 915e9fea6a1495f283a454f35238317a011093fd Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 13 Mar 2025 14:57:50 +0000 Subject: [PATCH 24/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index ccb9c4c..0866a7b 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -249,22 +249,33 @@ jobs: ### Azure upload - name: List of esisting blobs + id: removefiles uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | - dstprefix="dev/${{github.repository}}/previews/${PR_NUMBER}" - srcprefix="$DOCS_SOURCE_PATH" + az storage blob list \ --container-name '$web' \ --account-name ${{steps.keyvault.outputs.accountName}} \ --auth-mode login \ - --prefix "${dstprefix}/" > blobs.txt + --prefix "dev/${{github.repository}}/previews/${PR_NUMBER}/" > blobs.txt + + - name: Remove excessive files + if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' + run: | + dstprefix="dev/${{github.repository}}/previews/${PR_NUMBER}" + srcprefix="$DOCS_SOURCE_PATH" cat blobs.txt | grep name | cut -d: -f 2 | cut -d'"' -f2 > blobs_paths.txt + echo "Files in storage: `wc -l blobs_paths.txt`" cat blobs_paths.txt | sed -e "s#${dstprefix}/##g" > blobs_names.txt + echo "Names in storage: `wc -l blobs_names.txt`" find ${srcprefix} -type f > local_paths.txt + echo "Files local: `wc -l local_paths.txt`" cat local_paths.txt | sed -e "s#${srcprefix}/##g" > local_names.txt + echo "Names local: `wc -l local_names.txt`" grep -Fxv -f local_names.txt blobs_names.txt > delete.txt + echo "Files to delete: `wc -l delete.txt`" for file in `cat delete.txt` ; do # we have to remove the file # az storage blob delete \ From 42a5849fa7b0d1f4d793b2688df16f2f3769a9aa Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 13 Mar 2025 15:04:18 +0000 Subject: [PATCH 25/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 0866a7b..0c93044 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -274,7 +274,7 @@ jobs: echo "Files local: `wc -l local_paths.txt`" cat local_paths.txt | sed -e "s#${srcprefix}/##g" > local_names.txt echo "Names local: `wc -l local_names.txt`" - grep -Fxv -f local_names.txt blobs_names.txt > delete.txt + grep -Fxv -f local_names.txt blobs_names.txt > delete.txt || true echo "Files to delete: `wc -l delete.txt`" for file in `cat delete.txt` ; do # we have to remove the file From 2a7e476a4f9dc9f93abf15df54c3ee0de81c72db Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 13 Mar 2025 15:12:31 +0000 Subject: [PATCH 26/28] Update docs-build-push.yml test --- .github/workflows/docs-build-push.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 0c93044..1653a36 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -248,6 +248,22 @@ jobs: make deploy ### Azure upload + - name: TEST UPLOAD EXCESSIVE FILES + uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 + if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' + with: + inlineScript: | + mkdir test + for i in 1 3 4 5 6 ; do echo file$i > test/file$i; done + az storage blob upload-batch \ + -s test \ + -d '$web' \ + --destination-path "dev/${{github.repository}}/previews/${PR_NUMBER}" \ + --account-name ${{steps.keyvault.outputs.accountName}} \ + --overwrite \ + --content-cache-control "max-age=3600" \ + --auth-mode login + - name: List of esisting blobs id: removefiles uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 @@ -276,6 +292,7 @@ jobs: echo "Names local: `wc -l local_names.txt`" grep -Fxv -f local_names.txt blobs_names.txt > delete.txt || true echo "Files to delete: `wc -l delete.txt`" + cat delete.txt for file in `cat delete.txt` ; do # we have to remove the file # az storage blob delete \ From 3c6ab8614c70806225bcb81f64584f4a01750997 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 13 Mar 2025 15:34:11 +0000 Subject: [PATCH 27/28] Update docs-build-push.yml --- .github/workflows/docs-build-push.yml | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 1653a36..8c801d9 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -264,24 +264,18 @@ jobs: --content-cache-control "max-age=3600" \ --auth-mode login - - name: List of esisting blobs - id: removefiles + - name: Remove excessive files uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' with: inlineScript: | - + dstprefix="dev/${{github.repository}}/previews/${PR_NUMBER}" + srcprefix="$DOCS_SOURCE_PATH" az storage blob list \ --container-name '$web' \ --account-name ${{steps.keyvault.outputs.accountName}} \ --auth-mode login \ --prefix "dev/${{github.repository}}/previews/${PR_NUMBER}/" > blobs.txt - - - name: Remove excessive files - if: github.event.action == 'synchronize' || github.event.action == 'opened' || env.DEPLOYMENT_ENV == 'preview' - run: | - dstprefix="dev/${{github.repository}}/previews/${PR_NUMBER}" - srcprefix="$DOCS_SOURCE_PATH" cat blobs.txt | grep name | cut -d: -f 2 | cut -d'"' -f2 > blobs_paths.txt echo "Files in storage: `wc -l blobs_paths.txt`" cat blobs_paths.txt | sed -e "s#${dstprefix}/##g" > blobs_names.txt @@ -295,14 +289,13 @@ jobs: cat delete.txt for file in `cat delete.txt` ; do # we have to remove the file - # az storage blob delete \ - # --container-name '$web' - # --name ${dstprefix}/${file} - # --account-name ${{steps.keyvault.outputs.accountName}} \ - # --auth-mode login + az storage blob delete \ + --container-name '$web' + --name ${dstprefix}/${file} + --account-name ${{steps.keyvault.outputs.accountName}} \ + --auth-mode login echo "${dstprefix}/${file} should be removed" - done - + done - name: Azure upload PR preview uses: azure/cli@089eac9d8cc39f5d003e94f8b65efc51076c9cbd # v2.1.0 From 58e1b16ff63db351fecfc4439199c71a1933d1b9 Mon Sep 17 00:00:00 2001 From: Ekaterina Kukushkina Date: Thu, 13 Mar 2025 15:41:43 +0000 Subject: [PATCH 28/28] Update docs-build-push.yml typo --- .github/workflows/docs-build-push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-build-push.yml b/.github/workflows/docs-build-push.yml index 8c801d9..4945ffb 100644 --- a/.github/workflows/docs-build-push.yml +++ b/.github/workflows/docs-build-push.yml @@ -290,8 +290,8 @@ jobs: for file in `cat delete.txt` ; do # we have to remove the file az storage blob delete \ - --container-name '$web' - --name ${dstprefix}/${file} + --container-name '$web' \ + --name ${dstprefix}/${file} \ --account-name ${{steps.keyvault.outputs.accountName}} \ --auth-mode login echo "${dstprefix}/${file} should be removed"