Revision 20d53904a4c2c0c14760d15e4917b7865e7411e5 authored by Mohamed Barakat on 26 August 2022, 10:33:41 UTC, committed by GitHub on 26 August 2022, 10:33:41 UTC
release-gap-package
#!/bin/sh
#
# ReleaseTools - a set of shells script for making GAP package releases
#
# Please always use the latest version of this tool, available from
# <https://github.com/gap-system/ReleaseTools>.
#
# Copyright (c) 2013-2020 Max Horn <max@quendi.de>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
set -e
######################################################################
#
# Usage information
#
help() {
cat <<EOF
Usage: $0 [OPTIONS]
A tool for making releases of GAP packages on GitHub.
Run this from within a git clone of your package repository, checked out
at the revision you want to release. This tool extracts relevant data
from the PackageInfo.g file, and performs the releases process.
Actions
-h, --help display this help text and exit
-p, --push perform the final push, completing the release [Default]
-P, --no-push do not perform the final push
-f, --force if a release with the same name already exists: overwrite it
--skip-existing-release if a release with the same name already exists: exit without error
Paths
--srcdir <path> path of directory containing PackageInfo.g [Default: current directory]
--tmpdir <path> path of temporary directory [Default: tmp subdirectory of srcdir]
--webdir <path> path of web directory [Default: gh-pages subdirectory of srcdir]
--update-script <file> path of the update script [Default: update.g in webdir]
Custom settings
--token <oauth> GitHub access token
Notes:
* The package name and version, the list of archive formats, and the GitHub repository
are extracted from PackageInfo.g.
* To learn how to create a GitHub access token, please consult
https://help.github.com/articles/creating-an-access-token-for-command-line-use/
* Without the --push option, all steps are performed, except for the final push
of the gh-pages changes. These changes are what make the release visible
to the GAP package distribution system.
* Please consult the README for more information.
EOF
exit 0
}
######################################################################
#
# Various little helper functions
# print notices in green
notice() {
printf '\033[32m%s\033[0m\n' "$*"
}
# print warnings in yellow
warning() {
printf '\033[33mWARNING: %s\033[0m\n' "$*"
}
# print error in red and exit
error() {
printf '\033[31mERROR: %s\033[0m\n' "$*"
exit 1
}
# check for uncommitted changes
verify_git_clean() {
git update-index --refresh || error "uncommitted changes detected"
git diff-index --quiet HEAD -- || error "uncommitted changes detected"
}
# execute GAP commands from a heredoc, and export its output in the shell
# variable `gap_output`; abort if GAP reported any errors
run_gap() {
# Invoke GAP. The final call to 'tr' removes newlines and is a workaround
# for people whose gap script is broken (by not honoring -A, thus loading
# Browse, which inserts that extra character). I am looking at you,
# Sebastian!
gap_output=$( \
(echo 'OnBreak:=function() Print("FATAL ERROR"); FORCE_QUIT_GAP(1); end;;' ; cat - ; echo ; echo "FORCE_QUIT_GAP(0);") \
| $GAP -A -q -b 2>&1 \
| tr -d '\r' )
mkdir -p "$TMP_DIR"
echo "$gap_output" > "$TMP_DIR/gap-error.log"
if echo "$gap_output" | grep -q '\(Error\|FATAL ERROR\|Syntax \)' ; then
error "there was an error running GAP, see $TMP_DIR/gap-error.log"
fi
}
# helper function for parsing GitHub's JSON output. Right now,
# we only extract the value of a single key at a time. This means
# we may end up parsing the same JSON data two times, but that
# doesn't really matter as it is tiny.
json_get_key() {
echo "$response" | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get("'"$1"'",""))'
}
# On Mac OS X, tar stores extended attributes in ._FOO files inside archives.
# Setting COPYFILE_DISABLE prevents that. See <http://superuser.com/a/260264>
export COPYFILE_DISABLE=1
######################################################################
#
# Command line processing
#
SRC_DIR="$PWD"
TMP_DIR="$PWD/tmp"
PackageInfoURL=
README_URL=
PUSH=yes
FORCE=no
ONLY_TARBALL=no
SKIP_EXISTING_RELEASE=no
while [ x"$1" != x ]; do
option="$1" ; shift
case "$option" in
-h | --help ) help ;;
--srcdir ) SRC_DIR="$1"; shift ;;
--webdir ) WEB_DIR="$1"; shift ;;
--tmpdir ) TMP_DIR="$1"; shift ;;
--update-script ) UPDATE_SCRIPT="$1"; shift ;;
--srcdir=*) SRC_DIR=${option#--srcdir=}; shift ;;
--webdir=*) WEB_DIR=${option#--webdir=}; shift ;;
--tmpdir=*) TMP_DIR=${option#--tmpdir=}; shift ;;
--update-script=*) UPDATE_SCRIPT=${option#--update-script=}; shift ;;
--token ) TOKEN="$1"; shift ;;
-p | --push ) PUSH=yes ;;
-P | --no-push ) PUSH=no ;;
-f | --force ) FORCE=yes ;;
--no-force ) FORCE=no ;;
--only-tarball ) ONLY_TARBALL=yes ;;
--skip-existing-release ) SKIP_EXISTING_RELEASE=yes ;;
-- ) break ;;
* ) error "unknown option '$option'" ;;
esac
done
######################################################################
#
# Some initial sanity checks
#
cd "$SRC_DIR"
if [ ! -f PackageInfo.g ] ; then
error "unable to read PackageInfo.g file, use --help for instructions"
fi
# Check for webdir presence
if [ "x$WEB_DIR" = x ] ; then
WEB_DIR="$SRC_DIR/gh-pages"
fi
if [ ! -d "$WEB_DIR" ] ; then
error "could not find 'webdir' with clone of your gh-pages branch"
fi
# Check for presence of the update script
if [ "x$UPDATE_SCRIPT" = x ] ; then
UPDATE_SCRIPT="$WEB_DIR/update.g"
fi
if [ ! -f "$UPDATE_SCRIPT" ] ; then
error "could not find update script \"${UPDATE_SCRIPT}\""
fi
# Check whether GAP is usable
GAP=${GAP:-gap}
command -v "$GAP" >/dev/null 2>&1 ||
error "could not find GAP (perhaps set the GAP environment variable?)"
command -v curl >/dev/null 2>&1 ||
error "the 'curl' command was not found, please install it"
command -v git >/dev/null 2>&1 ||
error "the 'git' command was not found, please install it"
command -v python >/dev/null 2>&1 ||
error "the 'python' command was not found, please install it"
verify_git_clean
######################################################################
#
# Determine package name and version, and other meta data
#
notice "Extracting information from PackageInfo.g..."
run_gap <<EOF
Read("PackageInfo.g");
Print("PKG=\"",GAPInfo.PackageInfoCurrent.PackageName,"\"\n");
Print("VERSION=\"",GAPInfo.PackageInfoCurrent.Version,"\"\n");
tmp:=GAPInfo.PackageInfoCurrent.Date;; # either YYYY-MM-DD or DD/MM/YYYY format
Print("DATE=\"",tmp,"\"\n");
Print("PackageInfoURL=\"",GAPInfo.PackageInfoCurrent.PackageInfoURL,"\"\n");
Print("README_URL=\"",GAPInfo.PackageInfoCurrent.README_URL,"\"\n");
# extract archive formats; these can be comma or space separated, we want
# to output them space separated
tmp := GAPInfo.PackageInfoCurrent.ArchiveFormats;;
tmp := JoinStringsWithSeparator(SplitString(tmp, ", "), " ");;
Print("ARCHIVE_FORMATS=\"",tmp,"\"\n");
tmp := SplitString(GAPInfo.PackageInfoCurrent.ArchiveURL, "/");;
if Length(tmp) = 9 and tmp{[1,2,3,6,7]} = ["https:","","github.com","releases","download"] then
Print("REPO=\"",Concatenation(tmp[4],"/",tmp[5]),"\"\n");
Print("TAG=\"",tmp[8],"\"\n");
len := Length(tmp[9]) - Length(GAPInfo.PackageInfoCurrent.Version);
if len > 0 then
Print("BASENAME=\"",tmp[9]{[1..len]},"\"\n");
fi;
else
tmp := GAPInfo.PackageInfoCurrent.ArchiveURL;
Print("GAP_ERROR=\"The ArchiveURL has unexpected value '",tmp,"'\"\n");
fi;
Print("PDFFile=\"",GAPInfo.PackageInfoCurrent.PackageDoc.PDFFile,"\"\n");
EOF
# evaluate the output of GAP, which should be valid shell script code
eval "$gap_output"
if [ x"$GAP_ERROR" != "x" ] ; then
error "$GAP_ERROR"
fi
notice "Package $PKG $VERSION"
case "$VERSION" in
*dev) error "Must not use 'dev' versions for package releases" ;;
esac
######################################################################
#
# Run makedoc.g (if present), as that also can uncover mistakes (e.g.
# for projects which generate doc/title.xml from PackageInfo.g, it is
# easy to forget that when making the release commit).
#
if [ -f makedoc.g ] ; then
notice "Building GAP package documentation (using makedoc.g)"
run_gap <<GAPInput
if not IsPackageMarkedForLoading("$PKG", "") then
SetPackagePath("$PKG", ".");
fi;
Read("makedoc.g");
GAPInput
elif [ -f doc/make_doc ] ; then
notice "Building GAP package documentation (using doc/make_doc)"
cd doc && ./make_doc && cd ..
else
warning "Could not build manual (no makedoc.g and no doc/make_doc found)"
fi
# check if building the manual modified any files
verify_git_clean
######################################################################
#
# Determine the basename for the package archives
#
#
if [ x"$BASENAME" = "x" ] ; then
BASENAME="$PKG-"
fi
BASENAME="$BASENAME$VERSION"
notice "Using archive basename $BASENAME"
######################################################################
#
# Fetch GitHub oauth token, used to authenticate the following commands.
# See https://help.github.com/articles/git-automation-with-oauth-tokens/
#
if [ "x$TOKEN" = x ] ; then
TOKEN=$(git config --get github.token || echo)
fi
if [ "x$TOKEN" = x ] && [ -r ~/.github_shell_token ] ; then
TOKEN=$(cat ~/.github_shell_token)
fi
if [ "x$TOKEN" = x ] ; then
error "could not determine GitHub access token, please consult the README"
fi
######################################################################
#
# Determine GitHub repository and username, and the current branch
#
notice "Using GitHub repository $REPO"
GITHUB_USER=$(dirname "$REPO")
notice "Using GitHub username $GITHUB_USER"
BRANCH=$(git symbolic-ref -q --short HEAD)
notice "Using branch $BRANCH"
######################################################################
#
# Derive API urls
#
API_URL=https://api.github.com/repos/$REPO/releases
UPLOAD_URL=https://uploads.github.com/repos/$REPO/releases
######################################################################
#
# Determine the tag
#
verify_git_clean
if git show-ref -q "$TAG" ; then
notice "Using git tag $TAG (derived from ArchiveURL in in PackageInfo.g)"
else
notice "Creating git tag $TAG"
git tag "$TAG"
fi;
######################################################################
#
# Check if a GitHub release for this tag already exists
#
response=$(curl -s -S -X GET "$API_URL/tags/$TAG" -H "Authorization: token $TOKEN")
MESSAGE=$(json_get_key message)
RELEASE_ID=$(json_get_key id)
if [ "$MESSAGE" = "Not Found" ] ; then
MESSAGE= # release does not yet exist -> that's how we like it
elif [ x"$RELEASE_ID" != x ] ; then
# release already exists -> skip, error out or delete it
if [ "x$SKIP_EXISTING_RELEASE" = xyes ] ; then
notice "release $TAG already exists on GitHub, skipping release"
exit 0
elif [ "x$FORCE" = xyes ] ; then
notice "Deleting existing release $TAG from GitHub"
response=$(curl --fail -s -S -X DELETE "$API_URL/$RELEASE_ID" -H "Authorization: token $TOKEN")
MESSAGE=
else
error "release $TAG already exists on GitHub, aborting (use --force to override this)"
fi
fi
if [ x"$MESSAGE" != x ] ; then
error "accessing GitHub failed: $MESSAGE"
fi
######################################################################
#
# Validate the tag
#
HEAD_REF=$(git rev-parse --verify HEAD)
TAG_REF=$(git rev-parse --verify "$TAG^{}")
if [ "x$TAG_REF" != "x$HEAD_REF" ] ; then
error "tag $TAG is not the HEAD commit -- did you tag the right commit?"
fi
echo ""
######################################################################
#
# Get fresh (unmodified) copies of the files, and generate some stuff
#
# Clean any remains of previous export attempts
mkdir -p "$TMP_DIR"
rm -rf "${TMP_DIR:?}/$BASENAME"*
# Set umask to ensure the file permissions in the release
# archives are sane.
umask 0022
notice "Exporting repository content for tag '$TAG'"
git archive --prefix="$BASENAME/" "$TAG" . | tar xf - -C "$TMP_DIR"
# Build the package documentation, run autoconf, etc.
cd "$TMP_DIR/$BASENAME"
# adjust date
# Note that we cannot use sed's `-i` option for in-place editing, as
# that is a non-portable extension of POSIX, which works differently in
# BSD and GNU make.
sed "s;Date := .*;Date := \"$(date +%d/%m/%Y)\",;" PackageInfo.g > PackageInfo.g.bak
mv PackageInfo.g.bak PackageInfo.g
notice "Removing unnecessary files"
# Remove recursively in case there is a .github directory
rm -rf .git* .hg* .cvs* .circleci
rm -f .appveyor.yml .codecov.yml .travis.yml
# execute .release script, if present
if [ -f .release ] ; then
# the .release script can perform additional preparation, e.g.:
# * add files for distribution which are not part of the repository;
# * remove further files not intended for distribution (e.g. scripts/ directory);
# * build the package manual in a custom way;
# * perform additional sanity checks;
# * ...
. ./.release
rm -f .release
fi
if [ -x autogen.sh ] ; then
notice "Generating build system files"
sh autogen.sh
rm -rf autom4te.cache
fi
if [ -f makedoc.g ] ; then
notice "Building GAP package documentation for archives (using makedoc.g)"
run_gap <<GAPInput
if not IsPackageMarkedForLoading("$PKG", "") then
SetPackagePath("$PKG", ".");
fi;
PushOptions(rec(relativePath:="../../.."));
Read("makedoc.g");
GAPInput
rm -f doc/*.tex
rm -f doc/*.aux doc/*.bbl doc/*.blg doc/*.brf doc/*.idx doc/*.ilg doc/*.ind doc/*.log doc/*.out doc/*.pnr doc/*.tst
elif [ -f doc/make_doc ] ; then
notice "Copying GAP package documentation for archives (using doc/make_doc)"
cp -r "$SRC_DIR/doc" .
cp -r "$SRC_DIR/htm" .
rm -f doc/*.aux doc/*.bbl doc/*.blg doc/*.brf doc/*.idx doc/*.ilg doc/*.ind doc/*.log doc/*.out doc/*.pnr doc/*.tst
fi
# make sure every file is readable
chmod -R a+r .
# replace links to packages which are possibly referenced in the documentation, keep this in sync with `Tests.yml.j2`
for package in CAP_project/CAP CAP_project/CompilerForCAP CAP_project/MonoidalCategories CAP_project/CartesianCategories CAP_project/FreydCategoriesForCAP HigherHomologicalAlgebra/ToolsForHigherHomologicalAlgebra homalg_project/homalg homalg_project/Modules Toposes; do
# adjust links to other manuals
# Note that we cannot use sed's `-i` option for in-place editing, as
# that is a non-portable extension of POSIX, which works differently in
# BSD and GNU make.
for f in ./*/*.htm* ; do
sed "s;href=\"/home/gap/.gap/pkg/$package/doc/;href=\"https://homalg-project.github.io/$package/doc/;g" "$f" > "$f.bak"
mv "$f.bak" "$f"
done
done
# basic sanity check
fgrep -q -r '<a href="/' */*.htm* &&
error "HTML manual contains absolute paths"
######################################################################
#
# Validate PackageInfo.g
#
echo ""
notice "Validating PackageInfo.g..."
run_gap <<GAPInput || error "validation failed"
if not ValidatePackageInfo("PackageInfo.g") then
FORCE_QUIT_GAP(1);
fi;
GAPInput
######################################################################
#
# Verify README_URL refers to an existing file
#
README_NAME=$(basename "$README_URL")
[ -f "$README_NAME" ] ||
error "No file named '$README_NAME' exists; is your README_URL valid?"
######################################################################
#
# Create all archive files requested by ARCHIVE_FORMATS
#
cd "$TMP_DIR"
echo ""
for EXT in $ARCHIVE_FORMATS ; do
ARCHIVENAME=$BASENAME$EXT
FULLNAME="$TMP_DIR/$ARCHIVENAME"
notice "Creating $ARCHIVENAME ..."
case $EXT in
.tar.gz) tar cf - "$BASENAME" | gzip -9c > "$ARCHIVENAME" ;;
.tar.bz2) tar cf - "$BASENAME" | bzip2 -9c > "$ARCHIVENAME" ;;
.zip) zip -r9 --quiet "$ARCHIVENAME" "$BASENAME" ;;
*)
warning "unsupported archive format $EXT"
continue
;;
esac
if [ ! -f "$FULLNAME" ] ; then
error "failed creating $FULLNAME"
fi
done
######################################################################
#
# Abort early if user requested only creation of tarballs
#
if [ "x$ONLY_TARBALL" = xyes ] ; then
exit 0
fi
######################################################################
#
# Push commits to GitHub
#
cd "$SRC_DIR"
# construct GitHub URL for pushing
REMOTE="https://$GITHUB_USER:$TOKEN@github.com/$REPO"
# Make sure the branch is on the server
notice "Pushing your branch to GitHub"
git push "$REMOTE" "$BRANCH"
# Make sure the tag is on the server
notice "Pushing your tag to GitHub"
if [ "x$FORCE" = xyes ] ; then
git push --force "$REMOTE" "$TAG"
else
git push "$REMOTE" "$TAG"
fi
######################################################################
#
# Create the GitHub release
#
# Create the release by sending suitable JSON
DATA=$(cat <<EOF
{
"tag_name": "$TAG",
"name": "$PKG $VERSION",
"body": "Release for $PKG",
"draft": false,
"prerelease": false
}
EOF
)
notice "Creating new release $TAG on GitHub"
response=$(curl -s -S -H "Content-Type: application/json" \
-X POST --data "$DATA" "$API_URL" -H "Authorization: token $TOKEN")
MESSAGE=$(json_get_key message)
if [ x"$MESSAGE" != x ] ; then
error "creating release on GitHub failed: $MESSAGE"
fi
RELEASE_ID=$(json_get_key id)
if [ x"$RELEASE_ID" = x ] ; then
error "creating release on GitHub failed: no release id"
fi
######################################################################
#
# Upload all archive files requested by ARCHIVE_FORMATS
#
cd "$TMP_DIR"
echo ""
for EXT in $ARCHIVE_FORMATS ; do
ARCHIVENAME=$BASENAME$EXT
FULLNAME="$TMP_DIR/$ARCHIVENAME"
case $EXT in
.tar.gz) MIMETYPE="application/x-gzip" ;;
.tar.bz2) MIMETYPE="application/x-bzip2" ;;
.zip) MIMETYPE="application/zip" ;;
*) continue ;;
esac
if [ ! -f "$FULLNAME" ] ; then
error "failed creating $FULLNAME"
fi
notice "Uploading $ARCHIVENAME with mime type $MIMETYPE"
response=$(curl --fail --progress-bar -o "$TMP_DIR/upload.log" \
-X POST "$UPLOAD_URL/$RELEASE_ID/assets?name=$ARCHIVENAME" \
-H "Accept: application/vnd.github.v3+json" \
-H "Authorization: token $TOKEN" \
-H "Content-Type: $MIMETYPE" \
--data-binary @"$FULLNAME")
done
######################################################################
#
# Upload PDF
#
cd "$TMP_DIR"
echo ""
FULLNAME="$TMP_DIR/$BASENAME/$PDFFile"
if [ ! -f "$FULLNAME" ] ; then
error "could not find PDF"
fi
notice "Uploading PDF"
response=$(curl --fail --progress-bar -o "$TMP_DIR/upload_pdf.log" \
-X POST "$UPLOAD_URL/$RELEASE_ID/assets?name=$BASENAME.pdf" \
-H "Accept: application/vnd.github.v3+json" \
-H "Authorization: token $TOKEN" \
-H "Content-Type: application/pdf" \
--data-binary @"$FULLNAME")
######################################################################
#
# Update the website
#
notice "Updating website"
cd "$WEB_DIR"
git pull --ff-only
cp "$TMP_DIR/$BASENAME/$README_NAME" .
cp "$TMP_DIR/$BASENAME/PackageInfo.g" .
rm -rf doc/ htm/
if [ -f "$TMP_DIR/$BASENAME/doc/chap0.html" ] ; then
mkdir -p doc/
for ext in css html js txt ; do
cp "$TMP_DIR/$BASENAME/doc"/*.$ext doc/
done
fi
if [ -d "$TMP_DIR/$BASENAME/htm" ] ; then
cp -r "$TMP_DIR/$BASENAME/htm" .
fi
# adjust links to the GAP manuals
# Note that we cannot use sed's `-i` option for in-place editing, as
# that is a non-portable extension of POSIX, which works differently in
# BSD and GNU make.
for f in ./*/*.htm* ; do
sed 's;href="../../../doc/;href="https://www.gap-system.org/Manuals/doc/;g' "$f" > "$f.bak"
mv "$f.bak" "$f"
done
run_gap <<GAPInput
Read("$UPDATE_SCRIPT");
GAPInput
git add -A
git commit -m "Update website for $PKG $VERSION"
if [ "x$PUSH" = xyes ] ; then
notice "Pushing website changes"
git push "$REMOTE"
notice "Done"
notice "Your PackageInfo.g is now at $PackageInfoURL"
else
warning "To complete your release, run 'git push' in your gh-pages directory"
notice "Afterwards, your PackageInfo.g will be at $PackageInfoURL"
fi
exit 0
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...