Revision 3fe396c2fc041dad61503c83018c459efc0a91dc authored by Kirill Lakhov on 11 April 2024, 13:29:28 UTC, committed by GitHub on 11 April 2024, 13:29:28 UTC
<!-- Raise an issue to propose your change (https://github.com/opencv/cvat/issues). It helps to avoid duplication of efforts from multiple independent contributors. Discuss your ideas with maintainers to be sure that changes will be approved and merged. Read the [Contribution guide](https://opencv.github.io/cvat/docs/contributing/). --> <!-- Provide a general summary of your changes in the Title above --> ### Motivation and context <!-- Why is this change required? What problem does it solve? If it fixes an open issue, please link to the issue here. Describe your changes in detail, add screenshots. --> This PR allows users to upload `.rar` archives during task creation. It fixes several problems: 1. Outdated mimetype for `.rar` type which is now `x-rar-compressed` 2. Adds usage of `patool` util for `pyunpack` package, which previously used only `zipfile`. Patool can handle different kinds of archives, current supported list(by mimetype) can be found in the `supportedArchives` 1. The pipeline for rar archives now is `pyunpack` -> `patool` -> `unrar` ### How has this been tested? <!-- Please describe in detail how you tested your changes. Include details of your testing environment, and the tests you ran to see how your change affects other areas of the code, etc. --> ### Checklist <!-- Go over all the following points, and put an `x` in all the boxes that apply. If an item isn't applicable for some reason, then ~~explicitly strikethrough~~ the whole line. If you don't do that, GitHub will show incorrect progress for the pull request. If you're unsure about any of these, don't hesitate to ask. We're here to help! --> - [x] I submit my changes into the `develop` branch - [ ] I have created a changelog fragment <!-- see top comment in CHANGELOG.md --> - [ ] I have updated the documentation accordingly - [ ] I have added tests to cover my changes - [x] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - ~~[ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/opencv/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/opencv/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/opencv/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/opencv/cvat/tree/develop/cvat-ui#versioning))~~ ### License - [x] I submit _my code changes_ under the same [MIT License]( https://github.com/opencv/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern.
1 parent 90d5e1f
schedule.yml
name: CI-nightly
on:
schedule:
- cron: '0 22 * * *'
workflow_dispatch:
env:
SERVER_IMAGE_TEST_REPO: cvat_server
UI_IMAGE_TEST_REPO: instrumentation_cvat_ui
CYPRESS_VERIFY_TIMEOUT: 180000 # https://docs.cypress.io/guides/guides/command-line#cypress-verify
jobs:
check_updates:
runs-on: ubuntu-latest-big
env:
REPO: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
outputs:
last_commit_time: ${{ steps.check_updates.outputs.last_commit_time }}
last_night_time: ${{ steps.check_updates.outputs.last_night_time }}
steps:
- id: check_updates
run: |
default_branch=$(gh api /repos/$REPO | jq -r '.default_branch')
last_commit_date=$(gh api /repos/${REPO}/branches/${default_branch} | jq -r '.commit.commit.author.date')
last_night_date=$(gh api /repos/${REPO}/actions/workflows/schedule.yml/runs | \
jq -r '.workflow_runs[]? | select((.status == "completed")) | .updated_at' \
| sort | tail -1)
last_night_time=$(date +%s -d $last_night_date)
last_commit_time=$(date +%s -d $last_commit_date)
echo Last CI-nightly workflow run time: $last_night_date
echo Last commit time in develop branch: $last_commit_date
echo "last_commit_time=${last_commit_time}" >> $GITHUB_OUTPUT
echo "last_night_time=${last_night_time}" >> $GITHUB_OUTPUT
search_cache:
needs: check_updates
uses: ./.github/workflows/search-cache.yml
build:
needs: search_cache
runs-on: ubuntu-latest-big
steps:
- uses: actions/checkout@v4
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_CI_USERNAME }}
password: ${{ secrets.DOCKERHUB_CI_TOKEN }}
- name: CVAT server. Getting cache from the default branch
uses: actions/cache@v3
with:
path: /tmp/cvat_cache_server
key: ${{ runner.os }}-build-server-${{ needs.search_cache.outputs.sha }}
- name: CVAT UI. Getting cache from the default branch
uses: actions/cache@v3
with:
path: /tmp/cvat_cache_ui
key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }}
- name: CVAT server. Extract metadata (tags, labels) for Docker
id: meta-server
uses: docker/metadata-action@master
with:
images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }}
tags:
type=raw,value=nightly
- name: CVAT UI. Extract metadata (tags, labels) for Docker
id: meta-ui
uses: docker/metadata-action@master
with:
images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.UI_IMAGE_TEST_REPO }}
tags:
type=raw,value=nightly
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: CVAT server. Build and push
uses: docker/build-push-action@v3
with:
cache-from: type=local,src=/tmp/cvat_cache_server
context: .
file: Dockerfile
push: true
tags: ${{ steps.meta-server.outputs.tags }}
labels: ${{ steps.meta-server.outputs.labels }}
- name: CVAT UI. Build and push
uses: docker/build-push-action@v3
with:
cache-from: type=local,src=/tmp/cvat_cache_ui
context: .
file: Dockerfile.ui
push: true
tags: ${{ steps.meta-ui.outputs.tags }}
labels: ${{ steps.meta-ui.outputs.labels }}
unit_testing:
needs: build
runs-on: ubuntu-latest-big
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Getting CVAT UI cache from the default branch
uses: actions/cache@v3
with:
path: /tmp/cvat_cache_ui
key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }}
- name: Building CVAT UI image
uses: docker/build-push-action@v2
with:
context: .
file: ./Dockerfile.ui
cache-from: type=local,src=/tmp/cvat_cache_ui
tags: cvat/ui:latest
load: true
- name: CVAT server. Extract metadata (tags, labels) for Docker
id: meta-server
uses: docker/metadata-action@master
with:
images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }}
tags:
type=raw,value=nightly
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_CI_USERNAME }}
password: ${{ secrets.DOCKERHUB_CI_TOKEN }}
- name: Pull CVAT server image
run: |
docker pull ${{ steps.meta-server.outputs.tags }}
docker tag ${{ steps.meta-server.outputs.tags }} cvat/server:local
docker tag ${{ steps.meta-server.outputs.tags }} cvat/server:latest
docker tag cvat/ui:latest cvat/ui:local
- name: OPA tests
run: |
curl -L -o opa https://openpolicyagent.org/downloads/v0.45.0/opa_linux_amd64_static
chmod +x ./opa
./opa test cvat/apps/iam/rules
- name: REST API and SDK tests
run: |
pip3 install -r cvat-sdk/gen/requirements.txt
./cvat-sdk/gen/generate.sh
pip3 install -r ./tests/python/requirements.txt
pip3 install -e ./cvat-sdk
pip3 install -e ./cvat-cli
pytest tests/python/
pytest tests/python/ --stop-services
- name: Unit tests
env:
HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
run: |
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server cvat_db
max_tries=12
while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-c 'python manage.py test cvat/apps -v 2'
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-c 'DISABLE_HUSKY=1 yarn --frozen-lockfile && yarn workspace cvat-core run test'
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml down -v
e2e_testing:
needs: build
runs-on: ubuntu-latest-big
strategy:
fail-fast: false
matrix:
specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3',
'actions_objects', 'actions_objects2', 'actions_users',
'actions_projects_models', 'canvas3d_functionality', 'canvas3d_functionality_2',
'issues_prs', 'issues_prs2', 'features']
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '16.x'
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_CI_USERNAME }}
password: ${{ secrets.DOCKERHUB_CI_TOKEN }}
- name: CVAT server. Extract metadata (tags, labels) for Docker
id: meta-server
uses: docker/metadata-action@master
with:
images: ${{ secrets.DOCKERHUB_CI_WORKSPACE }}/${{ env.SERVER_IMAGE_TEST_REPO }}
tags:
type=raw,value=nightly
- name: CVAT UI. Extract metadata (tags, labels) for Docker
id: meta-ui
uses: docker/metadata-action@master
with:
images: ${{ secrets.DOCKERHUB_CI_USERNAME }}/${{ env.UI_IMAGE_TEST_REPO }}
tags:
type=raw,value=nightly
- name: Pull CVAT UI image
run: |
docker pull ${{ steps.meta-server.outputs.tags }}
docker tag ${{ steps.meta-server.outputs.tags }} cvat/server:dev
docker pull ${{ steps.meta-ui.outputs.tags }}
docker tag ${{ steps.meta-ui.outputs.tags }} cvat/ui:dev
- name: Run CVAT instance
run: |
docker compose \
-f docker-compose.yml \
-f docker-compose.dev.yml \
-f tests/docker-compose.file_share.yml \
-f tests/docker-compose.minio.yml \
-f components/serverless/docker-compose.serverless.yml up -d
- name: Waiting for server
id: wait-server
env:
API_ABOUT_PAGE: "localhost:8080/api/server/about"
run: |
max_tries=60
status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE})
while [[ $status_code != "200" && max_tries -gt 0 ]]
do
echo Number of attempts left: $max_tries
echo Status code of response: $status_code
sleep 5
status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE})
(( max_tries-- ))
done
if [[ $status_code != "200" ]]; then
echo Response from server is incorrect, output:
cat /tmp/server_response
fi
echo "status_code=${status_code}" >> $GITHUB_OUTPUT
- name: Fail on bad response from server
if: steps.wait-server.outputs.status_code != '200'
uses: actions/github-script@v3
with:
script: |
core.setFailed('Workflow failed: incorrect response from server. See logs artifact to get more info')
- name: Add user for tests
env:
DJANGO_SU_NAME: "admin"
DJANGO_SU_EMAIL: "admin@localhost.company"
DJANGO_SU_PASSWORD: "12qwaszx"
run: |
docker exec -i cvat_server /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
- name: Run tests
run: |
cd ./tests
yarn --frozen-lockfile
shopt -s extglob
if [[ ${{ matrix.specs }} == canvas3d_* ]]; then
npx cypress run \
--headed \
--browser chrome \
--env coverage=false \
--config-file cypress_canvas3d.config.js \
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
else
npx cypress run \
--browser chrome \
--env coverage=false \
--spec 'cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js'
fi
- name: Creating a log file from "cvat" container logs
if: failure()
run: |
docker logs cvat_server > ${{ github.workspace }}/tests/cvat.log
- name: Uploading cypress screenshots as an artifact
if: failure()
uses: actions/upload-artifact@v3.1.1
with:
name: cypress_screenshots
path: ${{ github.workspace }}/tests/cypress/screenshots
- name: Uploading "cvat" container logs as an artifact
if: failure()
uses: actions/upload-artifact@v3.1.1
with:
name: cvat_container_logs
path: ${{ github.workspace }}/tests/cvat.log
Computing file changes ...