Skip to main content

Browse the archive

Revision 0535d452dd426ba62d6d312766906ddd1230c7eb authored by Andrey Zhavoronkov on 02 November 2023, 15:59:39 UTC, committed by GitHub on 02 November 2023, 15:59:39 UTC 
Chunk preparation optimization (#7081)
 
This PR speeds up the preparation of chunks by: 
1. loading images once instead of twice in each writer,
2. as well as by allowing simultaneous preparation of more than 1 chunk
using multithreading.
This allows to reduce the time for preparation of chunks for 4895 images
from 0:04:36 to 0:01:20 in case of preparation of 3 chunks in parallel
and 0:02:46 in case of 1 chunk in my environment.

Co-authored-by: Maria Khrustaleva <maya17grd@gmail.com>
1 parent 1f8d5d3
Raw File
lint-staged.config.js 
// lint-staged.config.js

const micromatch = require('micromatch');

function containsInPath(pattern, list) {
    return list.filter((item) => micromatch.contains(item, pattern));
}

function makePattern(extension) {
    return `**/*.${extension}`;
}

module.exports = (stagedFiles) => {
    const eslintExtensions = ['ts', 'tsx', 'js'].map(makePattern);
    const scssExtensions = ['scss'].map(makePattern);
    const eslintFiles = micromatch(stagedFiles, eslintExtensions);
    const scssFiles = micromatch(stagedFiles, scssExtensions);

    const tests = containsInPath('/tests/cypress', eslintFiles);
    const cvatData = containsInPath('/cvat-data/', eslintFiles);
    const cvatCore = containsInPath('/cvat-core/src', eslintFiles);
    const cvatCanvas = containsInPath('/cvat-canvas/', eslintFiles);
    const cvatCanvas3d = containsInPath('/cvat-canvas3d/', eslintFiles);
    const cvatUI = containsInPath('/cvat-ui/', eslintFiles);

    const mapping = {};
    const commands = [];
    mapping['npx stylelint --fix '] = scssFiles.join(' ');
    mapping['yarn run precommit:cvat-tests -- '] = tests.join(' ');
    mapping['yarn run precommit:cvat-ui -- '] = cvatUI.join(' ');
    mapping['yarn run precommit:cvat-data -- '] = cvatData.join(' ');
    mapping['yarn run precommit:cvat-core -- '] = cvatCore.join(' ');
    mapping['yarn run precommit:cvat-canvas -- '] = cvatCanvas.join(' ');
    mapping['yarn run precommit:cvat-canvas3d -- '] = cvatCanvas3d.join(' ');

    for (const command of Object.keys(mapping)) {
        const files = mapping[command];
        if (files.length) {
            commands.push(`${command} ${files}`);
        }
    }

    return commands;
};
back to top