Revision fb87d7dff80139f56f3dd3c37b58e1e9cc79f02d authored by Andrey Zhavoronkov on 14 June 2023, 21:13:38 UTC, committed by GitHub on 14 June 2023, 21:13:38 UTC
<!-- Raise an issue to propose your change (https://github.com/opencv/cvat/issues). It helps to avoid duplication of efforts from multiple independent contributors. Discuss your ideas with maintainers to be sure that changes will be approved and merged. Read the [Contribution guide](https://opencv.github.io/cvat/docs/contributing/). --> <!-- Provide a general summary of your changes in the Title above --> ### Motivation and context <!-- Why is this change required? What problem does it solve? If it fixes an open issue, please link to the issue here. Describe your changes in detail, add screenshots. --> ### How has this been tested? <!-- Please describe in detail how you tested your changes. Include details of your testing environment, and the tests you ran to see how your change affects other areas of the code, etc. --> Manually with docker-compose.https.yml config enabled ### Checklist <!-- Go over all the following points, and put an `x` in all the boxes that apply. If an item isn't applicable for some reason, then ~~explicitly strikethrough~~ the whole line. If you don't do that, GitHub will show incorrect progress for the pull request. If you're unsure about any of these, don't hesitate to ask. We're here to help! --> - [x] I submit my changes into the `develop` branch - [ ] I have added a description of my changes into the [CHANGELOG](https://github.com/opencv/cvat/blob/develop/CHANGELOG.md) file - [ ] I have updated the documentation accordingly - [ ] I have added tests to cover my changes - [ ] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - [ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/opencv/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/opencv/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/opencv/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/opencv/cvat/tree/develop/cvat-ui#versioning)) ### License - [x] I submit _my code changes_ under the same [MIT License]( https://github.com/opencv/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. --------- Co-authored-by: Nikita Manovich <nikita@cvat.ai>
1 parent b19e925
main.yml
name: CI
on:
push:
branches:
- 'master'
- 'develop'
pull_request:
types: [edited, ready_for_review, opened, synchronize, reopened]
paths-ignore:
- 'site/**'
- '**/*.md'
env:
CYPRESS_VERIFY_TIMEOUT: 180000 # https://docs.cypress.io/guides/guides/command-line#cypress-verify
CVAT_VERSION: "local"
jobs:
search_cache:
if: |
github.event.pull_request.draft == false &&
!startsWith(github.event.pull_request.title, '[WIP]') &&
!startsWith(github.event.pull_request.title, '[Dependent]')
uses: ./.github/workflows/search-cache.yml
build:
needs: search_cache
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: CVAT server. Getting cache from the default branch
uses: actions/cache@v3
with:
path: /tmp/cvat_cache_server
key: ${{ runner.os }}-build-server-${{ needs.search_cache.outputs.sha }}
- name: CVAT UI. Getting cache from the default branch
uses: actions/cache@v3
with:
path: /tmp/cvat_cache_ui
key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Create artifact directories
run: |
mkdir /tmp/cvat_server
mkdir /tmp/cvat_ui
mkdir /tmp/cvat_sdk
- name: CVAT server. Build and push
uses: docker/build-push-action@v3
with:
build-args: |
"COVERAGE_PROCESS_START=.coveragerc"
cache-from: type=local,src=/tmp/cvat_cache_server
context: .
file: Dockerfile
tags: cvat/server
outputs: type=docker,dest=/tmp/cvat_server/image.tar
- name: Instrumentation of the code then rebuilding the CVAT UI
run: |
yarn --frozen-lockfile
yarn run coverage
- name: CVAT UI. Build and push
uses: docker/build-push-action@v3
with:
cache-from: type=local,src=/tmp/cvat_cache_ui
context: .
file: Dockerfile.ui
tags: cvat/ui
outputs: type=docker,dest=/tmp/cvat_ui/image.tar
- name: CVAT SDK. Build
run: |
pip3 install --user -r cvat-sdk/gen/requirements.txt
./cvat-sdk/gen/generate.sh
cp -r cvat-sdk/* /tmp/cvat_sdk/
- name: Verify API schema
id: verify_schema
run: |
docker load --input /tmp/cvat_server/image.tar
docker run --rm --entrypoint /bin/bash cvat/server \
-c 'python manage.py spectacular' > cvat/schema-expected.yml
if ! git diff --no-index cvat/schema.yml cvat/schema-expected.yml; then
echo
echo 'API schema has changed! Please update cvat/schema.yml:'
echo
echo ' docker run --rm --entrypoint /bin/bash cvat/server:dev \'
echo " -c 'python manage.py spectacular' > cvat/schema.yml"
exit 1
fi
- name: Upload CVAT server artifact
uses: actions/upload-artifact@v3
with:
name: cvat_server
path: /tmp/cvat_server/image.tar
- name: Upload CVAT UI artifact
uses: actions/upload-artifact@v3
with:
name: cvat_ui
path: /tmp/cvat_ui/image.tar
- name: Upload CVAT SDK artifact
uses: actions/upload-artifact@v3
with:
name: cvat_sdk
path: /tmp/cvat_sdk/
rest_api_testing:
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Download CVAT server image
uses: actions/download-artifact@v3
with:
name: cvat_server
path: /tmp/cvat_server/
- name: Download CVAT UI images
uses: actions/download-artifact@v3
with:
name: cvat_ui
path: /tmp/cvat_ui/
- name: Load Docker images
run: |
docker load --input /tmp/cvat_server/image.tar
docker load --input /tmp/cvat_ui/image.tar
docker tag cvat/server:latest cvat/server:${CVAT_VERSION}
docker tag cvat/ui:latest cvat/ui:${CVAT_VERSION}
docker image ls -a
- name: Upload expected schema as an artifact
if: failure() && steps.verify_schema.conclusion == 'failure'
uses: actions/upload-artifact@v3.1.2
with:
name: expected_schema
path: cvat/schema-expected.yml
- name: Running REST API and SDK tests
id: run_tests
env:
COVERAGE_PROCESS_START: ".coveragerc"
run: |
pip3 install -r cvat-sdk/gen/requirements.txt
./cvat-sdk/gen/generate.sh
pip3 install -r ./tests/python/requirements.txt
pip3 install -e ./cvat-sdk
pip3 install -e ./cvat-cli
pytest tests/python/ --cov --cov-report xml
- name: Upload coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
- name: Creating a log file from cvat containers
if: failure() && steps.run_tests.conclusion == 'failure'
env:
LOGS_DIR: "${{ github.workspace }}/rest_api_testing"
run: |
mkdir $LOGS_DIR
docker logs test_cvat_server_1 > $LOGS_DIR/cvat_server.log
docker logs test_cvat_worker_export_1 > $LOGS_DIR/cvat_worker_export.log
docker logs test_cvat_worker_import_1 > $LOGS_DIR/cvat_worker_import.log
docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log
- name: Uploading "cvat" container logs as an artifact
if: failure() && steps.run_tests.conclusion == 'failure'
uses: actions/upload-artifact@v3.1.2
with:
name: rest_api_container_logs
path: "${{ github.workspace }}/rest_api_testing"
unit_testing:
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Download CVAT server image
uses: actions/download-artifact@v3
with:
name: cvat_server
path: /tmp/cvat_server/
- name: Load Docker server image
run: |
docker load --input /tmp/cvat_server/image.tar
docker tag cvat/server:latest cvat/server:${CVAT_VERSION}
docker image ls -a
- name: Running OPA tests
run: |
python cvat/apps/iam/rules/tests/generate_tests.py \
--output-dir cvat/apps/iam/rules/
curl -L -o opa https://openpolicyagent.org/downloads/v0.45.0/opa_linux_amd64_static
chmod +x ./opa
./opa test cvat/apps/iam/rules
- name: Running unit tests
env:
HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
run: |
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server cvat_db
max_tries=12
while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-c 'coverage run -a manage.py test cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}'
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test && mv cvat-core/reports/coverage/coverage-final.json ${CONTAINER_COVERAGE_DATA_DIR}'
- name: Upload coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
- name: Creating a log file from cvat containers
if: failure()
env:
LOGS_DIR: "${{ github.workspace }}/unit_testing"
run: |
mkdir $LOGS_DIR
docker logs cvat_server > $LOGS_DIR/cvat_server.log
docker logs cvat_opa 2> $LOGS_DIR/cvat_opa.log
- name: Uploading "cvat" container logs as an artifact
if: failure()
uses: actions/upload-artifact@v3.1.1
with:
name: unit_tests_container_logs
path: "${{ github.workspace }}/unit_testing"
e2e_testing:
needs: build
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3',
'actions_objects', 'actions_objects2', 'actions_users',
'actions_projects_models', 'actions_organizations', 'canvas3d_functionality',
'canvas3d_functionality_2', 'issues_prs', 'issues_prs2', 'masks', 'skeletons']
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '16.x'
- name: Download CVAT server image
uses: actions/download-artifact@v3
with:
name: cvat_server
path: /tmp/cvat_server/
- name: Download CVAT UI image
uses: actions/download-artifact@v3
with:
name: cvat_ui
path: /tmp/cvat_ui/
- name: Load Docker images
run: |
docker load --input /tmp/cvat_server/image.tar
docker load --input /tmp/cvat_ui/image.tar
docker tag cvat/server:latest cvat/server:${CVAT_VERSION}
docker tag cvat/ui:latest cvat/ui:${CVAT_VERSION}
docker image ls -a
- name: Run CVAT instance
run: |
docker compose \
-f docker-compose.yml \
-f docker-compose.dev.yml \
-f components/serverless/docker-compose.serverless.yml \
-f tests/docker-compose.minio.yml \
-f tests/docker-compose.file_share.yml up -d
- name: Waiting for server
env:
API_ABOUT_PAGE: "localhost:8080/api/server/about"
run: |
max_tries=60
status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE})
while [[ $status_code != "200" && max_tries -gt 0 ]]
do
echo Number of attempts left: $max_tries
echo Status code of response: $status_code
sleep 5
status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE})
(( max_tries-- ))
done
- name: Run E2E tests
env:
DJANGO_SU_NAME: 'admin'
DJANGO_SU_EMAIL: 'admin@localhost.company'
DJANGO_SU_PASSWORD: '12qwaszx'
run: |
docker exec -i cvat_server /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
cd ./tests
yarn --frozen-lockfile
if [[ ${{ matrix.specs }} == canvas3d_* ]]; then
npx cypress run \
--headed \
--browser chrome \
--config-file cypress_canvas3d.config.js \
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
else
npx cypress run \
--browser chrome \
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
fi
- name: Upload coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
- name: Creating a log file from "cvat" container logs
if: failure()
run: |
docker logs cvat_server > ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log
docker logs cvat_worker_export > ${{ github.workspace }}/tests/cvat_worker_export_${{ matrix.specs }}.log
docker logs cvat_worker_import > ${{ github.workspace }}/tests/cvat_worker_import_${{ matrix.specs }}.log
- name: Uploading "cvat" container logs as an artifact
if: failure()
uses: actions/upload-artifact@v3.1.1
with:
name: e2e_container_logs
path: ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log
- name: Uploading cypress screenshots as an artifact
if: failure()
uses: actions/upload-artifact@v3.1.1
with:
name: cypress_screenshots_${{ matrix.specs }}
path: ${{ github.workspace }}/tests/cypress/screenshots
generate_github_pages:
needs: [rest_api_testing, unit_testing, e2e_testing]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
fetch-depth: 0
- name: Download CVAT server images
uses: actions/download-artifact@v3
with:
name: cvat_server
path: /tmp/cvat_server/
- name: Download CVAT server images
uses: actions/download-artifact@v3
with:
name: cvat_sdk
path: /tmp/cvat_sdk/
- name: Load Docker images
run: |
docker load --input /tmp/cvat_server/image.tar
- name: Setup Hugo
uses: peaceiris/actions-hugo@v2
with:
hugo-version: '0.83.1'
extended: true
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: '16.x'
- name: Install npm packages
working-directory: ./site
run: |
npm ci
- name: Build docs
run: |
pip install -r site/requirements.txt
python site/process_sdk_docs.py --input-dir /tmp/cvat_sdk/docs/ --site-root site/
python site/build_docs.py
env:
HUGO_ENV: production
- name: Deploy
if: github.ref == 'refs/heads/develop'
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./public
force_orphan: true
publish_dev_images:
if: github.ref == 'refs/heads/develop'
needs: [rest_api_testing, unit_testing, e2e_testing, generate_github_pages]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Download CVAT server images
uses: actions/download-artifact@v3
with:
name: cvat_server
path: /tmp/cvat_server/
- name: Download CVAT UI images
uses: actions/download-artifact@v3
with:
name: cvat_ui
path: /tmp/cvat_ui/
- name: Load Docker images
run: |
docker load --input /tmp/cvat_server/image.tar
docker load --input /tmp/cvat_ui/image.tar
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Push to Docker Hub
env:
SERVER_IMAGE_REPO: ${{ secrets.DOCKERHUB_WORKSPACE }}/server
UI_IMAGE_REPO: ${{ secrets.DOCKERHUB_WORKSPACE }}/ui
run: |
docker tag cvat/server:latest "${SERVER_IMAGE_REPO}:dev"
docker push "${SERVER_IMAGE_REPO}:dev"
docker tag cvat/ui:latest "${UI_IMAGE_REPO}:dev"
docker push "${UI_IMAGE_REPO}:dev"
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...