blob: 893a6b1c7d7df9d859f8291bded2ed0914c39cc7 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
#!/usr/bin/env nix-shell
#! nix-shell -i bash -p bash coreutils curl jq nix
# vim: ft=sh sw=2 et
#
# This scripts scans the github terraform-providers repo for new releases,
# generates the corresponding nix code and finally generates an index of
# all the providers given in ./providers.txt.
set -euo pipefail
# the maximum number of attempts before giving up inside of GET and prefetch_github
readonly maxAttempts=30
GET() {
local url=$1
local retry=1
echo "fetching $url" >&2
while ! curl -#fL -u "$GITHUB_AUTH" "$url"; do
echo "The curl command has failed. Attempt $retry/${maxAttempts}" >&2
if [[ "${retry}" -eq "${maxAttempts}" ]]; then
exit 1
fi
retry=$(( retry + 1 ))
sleep 5
done
}
get_org_repos() {
local org=$1
local page=1
GET "https://api.github.com/orgs/$org/repos?per_page=100" | jq -r '.[].name'
}
get_repo_tags() {
local owner=$1
local repo=$2
GET "https://api.github.com/repos/$owner/$repo/git/refs/tags?per_page=100" | \
jq -r '.[].ref' | \
grep -v 'v\.' | \
cut -d '/' -f 3- | \
sort --version-sort
}
prefetch_github() {
local owner=$1
local repo=$2
local rev=$3
local retry=1
while ! nix-prefetch-url --unpack "https://github.com/$owner/$repo/archive/$rev.tar.gz"; do
echo "The nix-prefetch-url command has failed. Attempt $retry/${maxAttempts}" >&2
if [[ "${retry}" -eq "${maxAttempts}" ]]; then
exit 1
fi
retry=$(( retry + 1 ))
sleep 5
done
}
echo_entry() {
local owner=$1
local repo=$2
local rev=$3
local version=$(echo $3 | sed 's/^v//')
local sha256=$4
cat <<EOF
{
owner = "$owner";
repo = "$repo";
rev = "$rev";
version = "$version";
sha256 = "$sha256";
};
EOF
}
indent() { sed 's/^/ /'; }
add_repo() {
org="${1}"
repo="${2}"
echo "*** $repo ***"
name=$(echo "$repo" | cut -d - -f 3-)
last_tag=$(get_repo_tags "$org" "$repo" | tail -1)
last_tag_sha256=$(prefetch_github "$org" "$repo" "$last_tag")
{
echo " $name ="
echo_entry "$org" "$repo" "$last_tag" "$last_tag_sha256" | indent
} >> data.nix
}
## Main ##
cd "$(dirname "$0")"
if [[ -z "${GITHUB_AUTH:-}" ]]; then
cat <<'HELP'
Missing the GITHUB_AUTH env. This is required to work around the 60 request
per hour rate-limit.
Go to https://github.com/settings/tokens and create a new token with the
"public_repo" scope.
Then `export GITHUB_AUTH=<your user>:<your token>` and run this script again.
HELP
exit 1
fi
cat <<HEADER > data.nix
# Generated with ./update-all
{
HEADER
while read line; do
IFS=' ' read -r -a fields <<< "$line"
if [[ "${#fields[@]}" -eq 0 ]]; then
continue
fi
if [[ "${fields[0]}" = *"/"* ]]; then
org="$(echo "${fields[0]}" | cut -d/ -f1)"
repo="$(echo "${fields[0]}" | cut -d/ -f2)"
add_repo "${org}" "${repo}"
else
org="${fields[0]}"
repos=$(get_org_repos "$org")
if [[ "${#fields[@]}" -ge 2 ]] && [[ -n "${fields[1]}" ]]; then
repos="$( echo "${repos[@]}" | grep "${fields[1]}" )"
fi
if [[ "${#fields[@]}" -eq 3 ]] && [[ -n "${fields[2]}" ]]; then
repos="$( echo "${repos[@]}" | grep -v "${fields[2]}" )"
fi
repos="$( echo "${repos[@]}" | sort )"
for repo in $repos; do
add_repo "$org" "$repo"
done
fi
done < <(grep -v '^#\|^$' providers.txt)
cat <<FOOTER >> data.nix
}
FOOTER
echo Done.
|