source/snippets/waterfall-eicweb.sh

0001 #!/bin/sh
0002
0003 project="$1"
0004 pipeline="$2"
0005
0006 [ -z "$project" -o -z "$pipeline" ] && echo "Usage: $0 <project-id> <pipeline-id>" && exit 1
0007
0008 url="https://eicweb.phy.anl.gov/api/v4/projects"
0009
0010 dir=$(mktemp -d)
0011
0012 print_cmd() {
0013     echo "$@"
0014     "$@"
0015 }
0016
0017 fetch_pipeline() {
0018     project="${1%%:*}"
0019     pipeline="${1##*:}"
0020     jobs_url="$url/$project/pipelines/$pipeline/jobs"
0021     per_page=100
0022     page=1
0023
0024     while true; do
0025         file="$dir/jobs-$project-$pipeline-$page.json"
0026         # Fetch if the file doesn't exist
0027         [ -f "$dir/$file" ] || print_cmd curl -LfsS "$jobs_url?include_retried=true&per_page=$per_page&page=$page" -o "$file" || break
0028         jq -e "length < $per_page" "$file" > /dev/null && break
0029         page=$((page + 1))
0030     done
0031
0032     # Get the bridges, shouldn't be paginated
0033     bridges_url="$url/$project/pipelines/$pipeline/bridges"
0034
0035     # Obtain the child pipeline ids
0036     child_pipelines=$(curl -LfsS "$bridges_url" | jq -r '.[].downstream_pipeline | "\(.project_id):\(.id)"')
0037
0038     # Fetch child pipelines
0039     for child_pipeline in $child_pipelines; do
0040         echo "Fetching pipeline $child_pipeline"
0041         fetch_pipeline "$child_pipeline"
0042     done
0043
0044 }
0045
0046 echo "Fetching main pipeline"
0047 fetch_pipeline "$project:$pipeline"
0048
0049 # Finally output as trace.json
0050 jq \
0051 'map(['\
0052 'select(.started_at and .finished_at) | '\
0053 '{name: (.name), cat: "PERF", ph: "B", pid: .pipeline.id, tid: .id, ts: (.started_at | sub("\\.[0-9]+Z$"; "Z") | fromdate * 10e5)},'\
0054 '{name: (.name), cat: "PERF", ph: "E", pid: .pipeline.id, tid: .id, ts: (.finished_at | sub("\\.[0-9]+Z$"; "Z") | fromdate * 10e5)}'\
0055 ']) | flatten(1) | .[]' $dir/jobs-*-*.json | jq -s > trace.json
0056
0057 echo "Now upload trace.json to https://ui.perfetto.dev"