https://github.com/ICIJ/datashare
Tip revision: 02e2296445b8104b01a2369c63eafa3d86ef0042 authored by Pierre Romera on 17 July 2023, 09:49:46 UTC
[release] 11.5.0
[release] 11.5.0
Tip revision: 02e2296
config.yml
version: 2.1
parameters:
publish_backend_doc:
type: boolean
default: false
jobs:
build_back:
docker:
- image: cimg/openjdk:11.0.16
- image: docker.elastic.co/elasticsearch/elasticsearch:7.9.1
environment:
- "discovery.type=single-node"
- "cluster.name=datashare"
- "ES_JAVA_OPTS=-Xms256m -Xmx256m"
name: elasticsearch
- image: redis:4.0.1-alpine
name: redis
- image: postgres:11.2
name: postgres
environment:
POSTGRES_USER: dstest
POSTGRES_PASSWORD: test
POSTGRES_DB: dstest
working_directory: /tmp/datashare
environment:
MAVEN_OPTS: "-Xms512m -Xmx512m -Xss10M"
steps:
- checkout
# Download and cache dependencies
- restore_cache:
keys:
- v1-dependencies-{{ checksum "pom.xml" }}
- run:
name: create a bootstrap datashare DB for jooq code generation
command: |
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --no-check-certificate --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get install -y postgresql-client-11
PGPASSWORD=test psql -h postgres -U dstest dstest -c "CREATE DATABASE datashare;"
- run: sudo apt-get install -y tesseract-ocr-eng
- run: mvn validate
- run: mvn -pl commons-test -am install
- run: mvn -pl datashare-db liquibase:update
- run: mvn test
- run: mvn -pl datashare-app -pl datashare-nlp-corenlp -am install -Dmaven.test.skip=true
- when:
condition:
equal: [ master, << pipeline.git.branch >> ]
steps:
- run: .circleci/publish_doc_if_necessary.sh
- save_cache:
paths:
- ~/.m2
key: v1-dependencies-{{ checksum "pom.xml" }}
publish_doc:
docker:
- image: cimg/openjdk:11.0.16
- image: docker.elastic.co/elasticsearch/elasticsearch:7.9.1
environment:
- "discovery.type=single-node"
- "cluster.name=datashare"
- "ES_JAVA_OPTS=-Xms256m -Xmx256m"
name: elasticsearch
- image: postgres:11.2
name: postgres
environment:
POSTGRES_USER: dstest
POSTGRES_DB: dstest
POSTGRES_PASSWORD: test
working_directory: ~/datashare
environment:
MAVEN_OPTS: "-Xms512m -Xmx512m -Xss10M"
steps:
- checkout
- restore_cache:
keys:
- v1-dependencies-{{ checksum "pom.xml" }}
- run:
name: Prepare SSH directory
command: |
mkdir -p ~/.ssh
chmod 700 ~/.ssh
- add_ssh_keys:
fingerprints:
# This deploy key has read-write permission on ICIJ/datashare-docs repository
# @see https://app.circleci.com/settings/project/github/ICIJ/datashare/ssh
# @see https://github.com/ICIJ/datashare-docs/settings/keys
- "d0:5d:4a:ba:8a:55:2e:1f:6f:ad:57:c5:e0:21:ce:34"
- run:
name: Configure Git identity
command: |
git config --global user.name $CIRCLE_USERNAME
git config --global user.email "engineering@icij.org"
- run:
name: Checkout Datashare Doc repository
command: |
ssh-keyscan github.com >> ~/.ssh/known_hosts
git clone git@github.com:ICIJ/datashare-docs.git ~/datashare-docs
- run:
name: Make dist without frontend
command: mvn -pl datashare-dist package > mvn.out
- run:
name: Populate apigen index
command: ./doc/apigen/populate_apigen.sh
- run:
name: Launch datashare apigen
command: ./doc/apigen/datashare_for_apigen.sh
background: true
- run:
name: Install apigen
command: |
sudo apt-get update
sudo apt-get install -y python3-pip
sudo pip3 install fluent-http-apigen
- run:
name: Run apigen
command: apigen datashare-app/src/main/java/org/icij/datashare/web/*Resource.java > ~/datashare-docs/developers/backend/api.md
- run:
name: Install postgres
command: |
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --no-check-certificate --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get install -y postgresql-client-11
- run:
name: Generate db doc
environment:
PGDB: datashare
PGUSER: dstest
PGPASSWORD: test
command: |
echo "postgres:5432:$PGDB:$PGUSER:$PGPASSWORD" > ~/.pgpass && chmod 600 ~/.pgpass
createdb -U $PGUSER -h postgres $PGDB
mvn -pl commons-test -am install
mvn -pl datashare-db liquibase:update
./datashare-db/scr/build_db_doc.sh > ~/datashare-docs/developers/backend/database.md
- run:
name: Add/Commit changes to the doc
command: |
git -C ~/datashare-docs add -A
git -C ~/datashare-docs commit -am "doc: update backend documentation [${CIRCLE_SHA1}]" || true
- run:
name: Push changes to the doc
command: git -C ~/datashare-docs push origin main
package_back:
docker:
- image: cimg/openjdk:11.0.16
working_directory: /tmp/datashare
environment:
MAVEN_OPTS: "-Xms512m -Xmx512m -Xss10M"
steps:
- attach_workspace:
at: /tmp/datashare-client
- checkout
- restore_cache:
keys:
- v1-dependencies-{{ checksum "pom.xml" }}
- run:
name: make app link to front
command: ln -s /tmp/datashare-client/dist /tmp/datashare/app
- run:
name: make dist including frontend
command: mvn -pl datashare-dist package > mvn.out
- run:
name: make tarball for non debian linux
command: |
cp /tmp/datashare/datashare-dist/src/main/deb/bin/datashare /tmp/datashare/datashare-dist/target
cd /tmp/datashare/datashare-dist/target
tar cvzf datashare-dist-${CIRCLE_TAG}.tgz datashare datashare-dist-${CIRCLE_TAG}-all.jar
- persist_to_workspace:
root: /tmp/datashare
paths:
- datashare-dist
deploy_back:
docker:
- image: cimg/openjdk:11.0.16
working_directory: /tmp
steps:
- attach_workspace:
at: /tmp/datashare
- run:
name: deploy datashare dist jar to github release
command: |
upload_url=$(curl -X POST -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GITHUB_TOKEN" -d "{\"tag_name\":\"${CIRCLE_TAG}\", \"name\":\"${CIRCLE_TAG}\",\"body\":\"release ${CIRCLE_TAG}\"}" "https://api.github.com/repos/ICIJ/datashare/releases" | jq -r '.upload_url')
upload_url="${upload_url%\{*}"
echo "uploading asset to release url: $upload_url"
curl -X POST -s -m 120 -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GITHUB_TOKEN" -H "Content-Type: application/java-archive" --data-binary "@/tmp/datashare/datashare-dist/target/datashare-dist-${CIRCLE_TAG}-all.jar" "$upload_url?name=datashare-dist-${CIRCLE_TAG}-all.jar&label=datashare-dist-${CIRCLE_TAG}-all.jar"
curl -X POST -s -m 120 -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GITHUB_TOKEN" -H "Content-Type: application/java-archive" --data-binary "@/tmp/datashare/datashare-dist/target/datashare-dist-${CIRCLE_TAG}-back.jar" "$upload_url?name=datashare-dist-${CIRCLE_TAG}.jar&label=datashare-dist-${CIRCLE_TAG}.jar"
build_front:
resource_class: large
docker:
- image: cimg/node:16.16.0
- image: docker.elastic.co/elasticsearch/elasticsearch:7.9.1
environment:
- "discovery.type=single-node"
- "cluster.name=datashare"
- "ES_JAVA_OPTS=-Xms256m -Xmx256m"
- "http.cors.enabled=true"
- "http.cors.allow-origin=*"
- "http.cors.allow-methods=OPTIONS, HEAD, GET, POST, PUT, DELETE"
name: elasticsearch
working_directory: /tmp
steps:
- attach_workspace:
at: /tmp/datashare
- run:
name: adds_ssh_dir
command: |
mkdir -p ~/.ssh
chmod 700 ~/.ssh
- add_ssh_keys:
fingerprints:
- "b3:e6:7c:f3:86:9e:e9:88:d1:cf:22:8c:97:3e:93:54"
- run:
name: clone
command: |
ssh-keyscan github.com >> ~/.ssh/known_hosts
git clone git@github.com:ICIJ/datashare-client.git
- run:
name: release client on GitHub
command: |
upload_url=$(curl -X POST -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GITHUB_TOKEN" -d "{ \"tag_name\": \"${CIRCLE_TAG}\", \"target_commitish\": \"master\", \"name\": \"${CIRCLE_TAG}\", \"body\": \"Released by Circle CI\", \"draft\": false, \"prerelease\": true }" "https://api.github.com/repos/ICIJ/datashare-client/releases")
echo $upload_url > /tmp/datashare-client/datashare-client.url
- run:
name: checkout tag
command: |
cd datashare-client
git pull origin master --tags
git checkout ${CIRCLE_TAG}
- restore_cache:
keys:
- dependency-cache-{{ checksum "/tmp/datashare-client/package.json" }}
- dependency-cache-
- run:
name: make install
command: |
cd /tmp/datashare-client
make install
- save_cache:
key: dependency-cache-{{ checksum "/tmp/datashare-client/package.json" }}
paths:
- /tmp/datashare-client/node_modules
- run:
name: run tests and build dist
command: |
cd /tmp/datashare-client
make unit dist
- persist_to_workspace:
root: /tmp/datashare-client
paths:
- dist
- datashare-client.url
deploy_front:
docker:
- image: cimg/node:16.16.0
working_directory: /tmp
steps:
- attach_workspace:
at: /tmp/datashare
- run:
name: deploy datashare-client dist to github release
command: |
cd /tmp/datashare/dist
tar czf ../datashare-client-${CIRCLE_TAG}.tgz .
zip -r ../datashare-client-${CIRCLE_TAG}.zip .
upload_url=$(cat /tmp/datashare/datashare-client.url | jq -r '.upload_url')
if [ "$upload_url" = "null" ]; then
upload_url=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/ICIJ/datashare-client/releases/tags/${CIRCLE_TAG}" | jq -r '.upload_url')
fi
upload_url="${upload_url%\{*}"
echo "uploading asset to release url: $upload_url"
curl -X POST -s -m 120 -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GITHUB_TOKEN" -H "Content-Type: application/zip" --data-binary "@/tmp/datashare/datashare-client-${CIRCLE_TAG}.zip" "$upload_url?name=datashare-client-${CIRCLE_TAG}.zip&label=datashare-client-${CIRCLE_TAG}.zip"
curl -X POST -s -m 120 -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GITHUB_TOKEN" -H "Content-Type: application/gzip" --data-binary "@/tmp/datashare/datashare-client-${CIRCLE_TAG}.tgz" "$upload_url?name=datashare-client-${CIRCLE_TAG}.tgz&label=datashare-client-${CIRCLE_TAG}.tgz"
build_docker:
docker:
- image: docker:20.10.22-git
working_directory: /tmp
steps:
- attach_workspace:
at: /tmp/datashare
- setup_remote_docker
- run:
name: Login to Docker hub
command: |
docker login -u ${DOCKER_USER} -p ${DOCKER_PASS}
- run:
name: Install tonistiigi/binfmt support for additional platforms
command: |
docker run --privileged --rm tonistiigi/binfmt --install all
- run:
name: Build and push the image for several platform
command: |
cd /tmp/datashare/datashare-dist/target/datashare-dist-$CIRCLE_TAG-docker/
docker buildx create --use
docker buildx build \
--platform linux/amd64,linux/arm64 \
-t icij/datashare:${CIRCLE_TAG} \
--push .
build_installers:
resource_class: large
docker:
- image: circleci/build-image
working_directory: /tmp
steps:
- attach_workspace:
at: /tmp/datashare
- restore_cache:
keys:
- cargo-cache
- run:
name: Install dependencies
command: |
apt-get install -y nsis cpio jq imagemagick icnsutils
wget --no-check-certificate https://nsis.sourceforge.io/mediawiki/images/c/c9/Inetc.zip && unzip Inetc.zip && cp Plugins/x86-ansi/INetC.dll /usr/share/nsis/Plugins/
wget https://nsis.sourceforge.io/mediawiki/images/7/7f/EnVar_plugin.zip && unzip EnVar_plugin.zip && cp Plugins/x86-ansi/EnVar.dll /usr/share/nsis/Plugins/
wget https://github.com/hogliux/bomutils/archive/0.2.tar.gz && tar -zxf 0.2.tar.gz && cd bomutils-0.2 && make && make install
wget https://github.com/mackyle/xar/archive/refs/tags/xar-1.6.1.tar.gz && tar -zxf xar-1.6.1.tar.gz && cd xar-xar-1.6.1/xar && ./autogen.sh && make && make install
wget --no-check-certificate https://nsis.sourceforge.io/mediawiki/images/9/9d/Untgz.zip && unzip Untgz.zip && cp untgz/untgz.dll /usr/share/nsis/Plugins/
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
source "$HOME/.cargo/env" && cargo install apple-codesign
- save_cache:
key: cargo-cache
paths:
- $HOME/.cargo
- run:
name: Adds SSH directory
command: |
mkdir -p ~/.ssh
chmod 700 ~/.ssh
- add_ssh_keys:
fingerprints:
- "b3:e6:7c:f3:86:9e:e9:88:d1:cf:22:8c:97:3e:93:54"
- run:
name: Configure Git identity
command: |
git config --global user.name $CIRCLE_USERNAME
git config --global user.email "engineering@icij.org"
- run:
name: Checkout
command: |
ssh-keyscan github.com >> ~/.ssh/known_hosts
git clone git@github.com:ICIJ/datashare-installer.git
- run:
name: Build Mac installer
command: |
cd /tmp/datashare-installer/mac
source "$HOME/.cargo/env"
make VERSION=${CIRCLE_TAG} all
- run:
name: Build Windows installer
command: |
cd /tmp/datashare-installer/windows
make VERSION=${CIRCLE_TAG} all
- run:
name: Build Linux installer
command: |
cd /tmp/datashare-installer/linux
make VERSION=${CIRCLE_TAG} all
- run:
name: Populate VERSION.txt file
command: echo $CIRCLE_TAG > /tmp/datashare-installer/VERSION.txt
- run:
name: Commit version changes
command: |
git -C /tmp/datashare-installer/ commit -am "build: bump to ${CIRCLE_SHA1}" || true
- run:
name: Push changes
command: git -C /tmp/datashare-installer/ push origin master
- run:
name: Create git tag
command: |
curl -s -H "Authorization: token $GITHUB_TOKEN" -d "{\"tag_name\":\"${CIRCLE_TAG}\", \"name\":\"${CIRCLE_TAG}\",\"body\":\"release ${CIRCLE_TAG}\",\"prerelease\":true}" "https://api.github.com/repos/ICIJ/datashare-installer/releases"
- run:
name: Deploy installers on github
command: |
cd /tmp/datashare-installer
./deploy.sh ${CIRCLE_TAG}
upload_url=$(curl -s -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/ICIJ/datashare-installer/releases/tags/${CIRCLE_TAG}" | jq -r '.upload_url')
upload_url="${upload_url%\{*}"
curl -X POST -s -m 120 -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GITHUB_TOKEN" -H "Content-Type: application/vnd.debian.binary-package" --data-binary "@/tmp/datashare/datashare-dist/target/datashare-dist-${CIRCLE_TAG}.deb" "$upload_url?name=datashare-${CIRCLE_TAG}.deb&label=datashare-${CIRCLE_TAG}.deb"
curl -X POST -s -m 120 -H "Accept: application/vnd.github.v3+json" -H "Authorization: token $GITHUB_TOKEN" -H "Content-Type: application/gzip" --data-binary "@/tmp/datashare/datashare-dist/target/datashare-dist-${CIRCLE_TAG}.tgz" "$upload_url?name=datashare-${CIRCLE_TAG}.tgz&label=datashare-${CIRCLE_TAG}.tgz"
workflows:
version: 2
build_workflow:
when:
not: << pipeline.parameters.publish_backend_doc >>
jobs:
- build_back:
filters:
tags:
only: /.*/
- package_back:
requires:
- build_back
- build_front
filters:
tags:
only: /^[0-9]+\..*/
branches:
ignore: /.*/
- deploy_back:
requires:
- package_back
filters:
tags:
only: /^[0-9]+\..*/
branches:
ignore: /.*/
- build_front:
requires:
- build_back
filters:
tags:
only: /^[0-9]+\..*/
branches:
ignore: /.*/
- deploy_front:
requires:
- build_front
filters:
tags:
only: /^[0-9]+\..*/
branches:
ignore: /.*/
- build_docker:
requires:
- package_back
filters:
tags:
only: /^[0-9]+\..*/
branches:
ignore: /.*/
- build_installers:
requires:
- build_docker
filters:
tags:
only: /^[0-9]+\..*/
branches:
ignore: /.*/
publish_doc_workflow:
when: false # << pipeline.parameters.publish_backend_doc >>
jobs:
- publish_doc