-
Notifications
You must be signed in to change notification settings - Fork 3.9k
[No QA] Add a worker to deploy community > help redirects #30739
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
a878d79
12836a4
96fc10e
863f1c4
1627a69
57de166
da796e0
3a902ac
e0e85ed
8556958
bb81e9c
96f426d
fffdf95
c793f15
eeb937d
616ddef
bd7ea22
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,5 @@ | ||
| # Every PR gets a review from an internal Expensify engineer | ||
| * @Expensify/pullerbear | ||
|
|
||
| # Every PR that touches redirects gets reviewed by ring0 | ||
| docs/redirects.csv @Expensify/infra | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Adds new routes to the Cloudflare Bulk Redirects list for communityDot to helpDot | ||
| # pages. Does some basic sanity checking. | ||
|
|
||
|
AndrewGable marked this conversation as resolved.
|
||
| set -e | ||
|
|
||
| source scripts/shellUtils.sh | ||
|
|
||
| info "Adding any new redirects from communityDot to helpDot" | ||
|
|
||
| declare -r LIST_ID="20eb13215038446a98fd69ccf6d1026d" | ||
| declare -r ZONE_ID="$CLOUDFLARE_ACCOUNT_ID" | ||
| declare -r REDIRECTS_FILE="docs/redirects.csv" | ||
|
|
||
| function checkCloudflareResult { | ||
| RESULTS=$1 | ||
| RESULT_MESSAGE=$(echo "$RESULTS" | jq .success) | ||
|
|
||
| if ! [[ "$RESULT_MESSAGE" == "true" ]]; then | ||
| ERROR_MESSAGE=$(echo "$RESULTS" | jq .errors) | ||
| error "Error calling Cloudfalre API: $ERROR_MESSAGE" | ||
|
coleaeason marked this conversation as resolved.
|
||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| declare -a ITEMS_TO_ADD | ||
|
|
||
| while read -r line; do | ||
| # Split each line of the file into a source and destination so we can sanity check | ||
| # and compare against the current list. | ||
| read -r -a LINE_PARTS < <(echo "$line" | tr ',' ' ') | ||
| SOURCE_URL=${LINE_PARTS[0]} | ||
| DEST_URL=${LINE_PARTS[1]} | ||
|
|
||
| # Make sure the format of the line is as execpted. | ||
| if [[ "${#LINE_PARTS[@]}" -gt 2 ]]; then | ||
| error "Found a line with more than one comma: $line" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Basic sanity checking to make sure that the source and destination are in expected | ||
| # subdomains. | ||
| if ! [[ $SOURCE_URL =~ ^https://community\.expensify\.com ]]; then | ||
| error "Found source URL that is not a community URL: $SOURCE_URL" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if ! [[ $DEST_URL =~ ^https://help\.expensify\.com ]]; then | ||
| error "Found destination URL that is not a help URL: $DEST_URL" | ||
| exit 1 | ||
| fi | ||
|
|
||
| info "Source: $SOURCE_URL and destination: $DEST_URL appear to be formatted correctly." | ||
|
|
||
| ITEMS_TO_ADD+=("$line") | ||
|
|
||
| # This line skips the first line in the csv because the first line is a header row. | ||
| done <<< "$(tail +2 $REDIRECTS_FILE)" | ||
|
|
||
| # Sanity check that we should actually be running this and we aren't about to delete | ||
| # every single redirect. | ||
| if [[ "${#ITEMS_TO_ADD[@]}" -lt 1 ]]; then | ||
| error "No items found to add, why are we running?" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # This block builds a single JSON object with all of our updates so we can | ||
| # reduce the number of API calls we make. You cannot add any logging or anything | ||
| # that prints to std out to this block or it will break. We capture all of the std out | ||
| # from this loop and pass it to jq to build the json object. Any non-json will break the | ||
| # jq call at the end. | ||
| PUT_JSON=$(for new in "${ITEMS_TO_ADD[@]}"; do | ||
| read -r -a LINE_PARTS < <(echo "$new" | tr ',' ' ') | ||
| SOURCE_URL=${LINE_PARTS[0]} | ||
| DEST_URL=${LINE_PARTS[1]} | ||
|
|
||
| # We strip the prefix here so that the rule will match both http and https. Since vanilla will eventially be removed, | ||
| # we need to catch both because we will not have the http > https redirect done by vanilla anymore. | ||
| NO_PREFIX_SOURCE_URL=${SOURCE_URL/https:\/\//} | ||
| jq -n --arg source "$NO_PREFIX_SOURCE_URL" --arg dest "$DEST_URL" '{"redirect": {source_url: $source, target_url: $dest}}' | ||
| done | jq -n '. |= [inputs]') | ||
|
|
||
| info "Adding redirects for $PUT_JSON" | ||
|
|
||
| # We use PUT here instead of POST so that we replace the entire list in place. This has many benefits: | ||
| # 1. We don't have to check if items are already in the list, allowing this script to run faster | ||
| # 2. We can support deleting redirects this way by simply removing them from the list | ||
| # 3. We can support updating redirects this way, in the case of typos or moved destinations. | ||
| # | ||
| # Additionally this API call is async, so after we finish it, we must poll to wait for it to finish to | ||
| # to know that it was actually completed. | ||
| PUT_RESULT=$(curl -s --request PUT --url "https://api.cloudflare.com/client/v4/accounts/$ZONE_ID/rules/lists/$LIST_ID/items" \ | ||
| --header 'Content-Type: application/json' \ | ||
| --header "Authorization: Bearer $CLOUDFLARE_LIST_TOKEN" \ | ||
| --data "$PUT_JSON") | ||
|
|
||
| checkCloudflareResult "$PUT_RESULT" | ||
| OPERATION_ID=$(echo "$PUT_RESULT" | jq -r .result.operation_id) | ||
|
|
||
| DONE=false | ||
|
|
||
| # Poll for completition | ||
|
coleaeason marked this conversation as resolved.
|
||
| while [[ $DONE == false ]]; do | ||
| CHECK_RESULT=$(curl -s --request GET --url "https://api.cloudflare.com/client/v4/accounts/$ZONE_ID/rules/lists/bulk_operations/$OPERATION_ID" \ | ||
| --header 'Content-Type: application/json' \ | ||
| --header "Authorization: Bearer $CLOUDFLARE_LIST_TOKEN") | ||
| checkCloudflareResult "$CHECK_RESULT" | ||
|
|
||
| STATUS=$(echo "$CHECK_RESULT" | jq -r .result.status) | ||
|
|
||
| # Exit on completed or failed, other options are pending or running, in both cases | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We don't need to add a max retry/time limit to this, right?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think so? We could but... i'm inclined to just let it run like this until it becomes a problem. Unless the Cloudflare API just never ran the job and left it queued forever, i think we will eventually exit here. |
||
| # we want to keep polling. | ||
| if [[ $STATUS == "completed" ]]; then | ||
| DONE=true | ||
| fi | ||
|
|
||
| if [[ $STATUS == "failed" ]]; then | ||
| ERROR_MESSAGE=$(echo "$CHECK_RESULT" | jq .result.error) | ||
| error "List update failed with error: $ERROR_MESSAGE" | ||
| exit 1 | ||
| fi | ||
| done | ||
|
|
||
| success "Updated lists successfully" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| name: Update Redirects to ExpensifyHelp | ||
|
|
||
| on: | ||
| # Run on any push to main that has changes to the redirects file | ||
| push: | ||
| branches: | ||
| - main | ||
| paths: | ||
| - 'docs/redirects.csv' | ||
|
|
||
| # Run on any manual trigger | ||
| workflow_dispatch: | ||
|
|
||
| # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nice, was about to ask about this 👍 |
||
| # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. | ||
| concurrency: | ||
| group: "redirects" | ||
| cancel-in-progress: false | ||
|
|
||
| jobs: | ||
| build: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 | ||
|
|
||
| - name: Create help dot redirect | ||
| env: | ||
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | ||
| CLOUDFLARE_LIST_TOKEN: ${{ secrets.CLOUDFLARE_LIST_TOKEN }} | ||
| run: ./.github/scripts/createHelpRedirects.sh | ||
This file was deleted.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| sourceURL,targetURL | ||
| https://community.expensify.com/discussion/5634/deep-dive-how-long-will-it-take-for-me-to-receive-my-reimbursement,https://help.expensify.com/articles/expensify-classic/get-paid-back/reports/Reimbursements |
Uh oh!
There was an error while loading. Please reload this page.