source/snippets/waterfall-eicweb.sh

0001 #!/bin/sh
0002
0003 PAGES=0
0004
0005 while [ $# -gt 0 ]; do
0006     case "$1" in
0007         --pages) PAGES=1; shift ;;
0008         *)       break ;;
0009     esac
0010 done
0011
0012 project="$1"
0013 pipeline="$2"
0014
0015 usage() {
0016     echo "Usage: $0 [--pages] <project-id> <pipeline-id>"
0017     echo ""
0018     echo "  --pages  Upload trace to eic/perfetto-launcher GitHub Pages and print the launcher URL"
0019     echo "           Requires GITHUB_PAGES_TOKEN env var with contents:write on eic/perfetto-launcher"
0020 }
0021
0022 [ -z "$project" -o -z "$pipeline" ] && usage && exit 1
0023
0024 url="https://eicweb.phy.anl.gov/api/v4/projects"
0025
0026 dir=$(mktemp -d)
0027
0028 print_cmd() {
0029     echo "$@"
0030     "$@"
0031 }
0032
0033 fetch_pipeline() {
0034     project="${1%%:*}"
0035     pipeline="${1##*:}"
0036     jobs_url="$url/$project/pipelines/$pipeline/jobs"
0037     per_page=100
0038     page=1
0039
0040     while true; do
0041         file="$dir/jobs-$project-$pipeline-$page.json"
0042         # Fetch if the file doesn't exist
0043         [ -f "$dir/$file" ] || print_cmd curl -LfsS "$jobs_url?include_retried=true&per_page=$per_page&page=$page" -o "$file" || break
0044         jq -e "length < $per_page" "$file" > /dev/null && break
0045         page=$((page + 1))
0046     done
0047
0048     # Get the bridges, shouldn't be paginated
0049     bridges_url="$url/$project/pipelines/$pipeline/bridges"
0050
0051     # Obtain the child pipeline ids; skip bridges where the child pipeline
0052     # has not yet been created (downstream_pipeline is null)
0053     child_pipelines=$(curl -LfsS "$bridges_url" | jq -r '.[] | select(.downstream_pipeline != null) | .downstream_pipeline | "\(.project_id):\(.id)"')
0054
0055     # Fetch child pipelines
0056     for child_pipeline in $child_pipelines; do
0057         echo "Fetching pipeline $child_pipeline"
0058         fetch_pipeline "$child_pipeline"
0059     done
0060
0061 }
0062
0063 main_pipeline="$pipeline"
0064 echo "Fetching main pipeline"
0065 fetch_pipeline "$project:$pipeline"
0066
0067 # Finally output as trace.json; skip jobs with null pipeline id (bridge
0068 # triggered but child pipeline not yet created when waterfall ran)
0069 jq \
0070 'map(['\
0071 'select(.started_at and .finished_at and .pipeline.id != null) | '\
0072 '{name: (.name), cat: "PERF", ph: "B", pid: .pipeline.id, tid: .id, ts: (.started_at | sub("\\.[0-9]+Z$"; "Z") | fromdate * 10e5)},'\
0073 '{name: (.name), cat: "PERF", ph: "E", pid: .pipeline.id, tid: .id, ts: (.finished_at | sub("\\.[0-9]+Z$"; "Z") | fromdate * 10e5)}'\
0074 ']) | flatten(1) | .[]' $dir/jobs-*-*.json | jq -s > trace.json
0075
0076 echo "Now upload trace.json to https://ui.perfetto.dev"
0077
0078 # Upload trace to eic/perfetto-launcher GitHub Pages and print the launcher URL
0079 upload_to_pages() {
0080     _pipeline="$1"
0081     _dest="traces/trace-${_pipeline}.json"
0082     _launcher_url="https://eic.github.io/perfetto-launcher/?trace=${_dest}"
0083
0084     if [ -z "$GITHUB_PAGES_TOKEN" ]; then
0085         echo "ERROR: GITHUB_PAGES_TOKEN is not set; cannot upload to GitHub Pages" >&2
0086         return 1
0087     fi
0088
0089     echo "Uploading trace to eic/perfetto-launcher as ${_dest}..."
0090
0091     # Fetch existing file sha if it exists (needed for updates via GitHub Contents API)
0092     _sha=$(curl -sS \
0093         -H "Authorization: token ${GITHUB_PAGES_TOKEN}" \
0094         "https://api.github.com/repos/eic/perfetto-launcher/contents/${_dest}" \
0095         | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('sha',''))" 2>/dev/null || true)
0096
0097     # Build the JSON body with Python to avoid shell "Argument list too long" on large traces
0098     _body_file=$(mktemp /tmp/gh_upload_XXXXXX)
0099     python3 - << PYEOF
0100 import json, base64
0101 with open('trace.json', 'rb') as f:
0102     content = base64.b64encode(f.read()).decode()
0103 sha = '${_sha}'
0104 body = {
0105     'message': ('Update' if sha else 'Add') + ' trace for pipeline ${_pipeline}',
0106     'content': content,
0107 }
0108 if sha:
0109     body['sha'] = sha
0110 with open('${_body_file}', 'w') as out:
0111     json.dump(body, out)
0112 PYEOF
0113
0114     _response=$(curl -sS -X PUT \
0115         -H "Authorization: token ${GITHUB_PAGES_TOKEN}" \
0116         -H "Content-Type: application/json" \
0117         "https://api.github.com/repos/eic/perfetto-launcher/contents/${_dest}" \
0118         -d "@${_body_file}" \
0119         -w "\n%{http_code}")
0120     rm -f "$_body_file"
0121     _http_code=$(echo "$_response" | tail -1)
0122     if [ "$_http_code" != "201" ] && [ "$_http_code" != "200" ]; then
0123         echo "ERROR: GitHub API returned HTTP ${_http_code}" >&2
0124         echo "$_response" | head -n -1 >&2
0125         return 1
0126     fi
0127
0128     echo "Trace uploaded successfully."
0129     echo "Launcher URL: ${_launcher_url}"
0130     echo "(traces/index.json is regenerated automatically by the deploy workflow)"
0131
0132     if [ -t 1 ]; then
0133         case "$(uname)" in
0134             Darwin) open "$_launcher_url" ;;
0135             Linux)  xdg-open "$_launcher_url" 2>/dev/null || true ;;
0136         esac
0137     fi
0138 }
0139
0140 if [ "$PAGES" = "1" ]; then
0141     upload_to_pages "$main_pipeline"
0142 fi