https://github.com/owncloud/core
Tip revision: 0d731383d0522bb6dd037adb1157824a6629ddc6 authored by Phil Davis on 27 July 2021, 07:55:07 UTC
Remove smokeTest tag from Scenario Outline that tests the 'resend new user password set' link
Remove smokeTest tag from Scenario Outline that tests the 'resend new user password set' link
Tip revision: 0d73138
.drone.star
dir = {
'base': '/drone',
'server': '/drone/src',
'federated': '/drone/federated',
}
config = {
'rocketchat': {
'channel': 'server',
'from_secret': 'public_rocketchat',
},
'branches': [
'master',
],
'dependencies': True,
'codestyle': True,
'phpstan': True,
'phan': True,
'javascript': True,
'litmus': True,
'dav': True,
'phpunit': {
'allDatabases' : {
'phpVersions': [
'7.2',
]
},
'reducedDatabases' : {
'phpVersions': [
'7.3',
'7.4',
],
'databases': [
'sqlite',
'mariadb:10.2',
]
},
'external-samba-windows' : {
'phpVersions': [
'7.2',
'7.4',
],
'databases': [
'sqlite',
],
'externalTypes': [
'samba',
'windows',
],
'coverage': True,
'extraCommandsBeforeTestRun': [
'ls -l /var/cache',
'mkdir /var/cache/samba',
'ls -l /var/cache',
'ls -l /var/cache/samba',
]
},
'external-other' : {
'phpVersions': [
'7.2',
'7.4',
],
'databases': [
'sqlite',
],
'externalTypes': [
'webdav',
'sftp',
'scality',
'owncloud',
],
'coverage': True,
}
},
'acceptance': {
'api': {
'suites': [
'apiAuth',
'apiAuthOcs',
'apiAuthWebDav',
'apiCapabilities',
'apiComments',
'apiFavorites',
'apiMain',
'apiProvisioning-v1',
'apiProvisioning-v2',
'apiProvisioningGroups-v1',
'apiProvisioningGroups-v2',
'apiShareCreateSpecialToRoot1',
'apiShareCreateSpecialToShares1',
'apiShareCreateSpecialToRoot2',
'apiShareCreateSpecialToShares2',
'apiSharees',
'apiShareManagementToRoot',
'apiShareManagementToShares',
'apiShareManagementBasicToRoot',
'apiShareManagementBasicToShares',
'apiShareOperationsToRoot1',
'apiShareOperationsToRoot2',
'apiShareOperationsToShares1',
'apiShareOperationsToShares2',
'apiSharePublicLink1',
'apiSharePublicLink2',
'apiShareReshareToRoot1',
'apiShareReshareToShares1',
'apiShareReshareToRoot2',
'apiShareReshareToShares2',
'apiShareReshareToRoot3',
'apiShareReshareToShares3',
'apiShareUpdateToRoot',
'apiShareUpdateToShares',
'apiTags',
'apiTranslation',
'apiTrashbin',
'apiTrashbinRestore',
'apiVersions',
'apiWebdavEtagPropagation1',
'apiWebdavEtagPropagation2',
'apiWebdavLocks',
'apiWebdavLocks2',
'apiWebdavLocks3',
'apiWebdavLocksUnlock',
'apiWebdavMove1',
'apiWebdavMove2',
'apiWebdavOperations',
'apiWebdavPreviews',
'apiWebdavProperties1',
'apiWebdavProperties2',
'apiWebdavUpload1',
'apiWebdavUpload2',
],
},
'apiNotifications': {
'suites': [
'apiSharingNotificationsToRoot',
'apiSharingNotificationsToShares',
],
'extraApps': {
'notifications': 'if [ -f "composer.json" ]; then composer install; fi',
},
},
'apiFederation': {
'suites': [
'apiFederationToRoot1',
'apiFederationToRoot2',
'apiFederationToShares1',
'apiFederationToShares2',
],
'federatedServerNeeded': True,
'federatedServerVersions': ['git', 'latest', '10.6.0'],
},
'cli': {
'suites': [
'cliBackground',
'cliLocalStorage',
'cliMain',
'cliProvisioning',
'cliTrashbin',
],
'emailNeeded': True,
},
'cliAppManagement': {
'suites': [
'cliAppManagement',
],
'testingRemoteSystem': False,
},
'cliExternalStorage': {
'suites': [
'cliExternalStorage',
],
'federatedServerNeeded': True,
'federatedServerVersions': ['git', 'latest', '10.6.0'],
},
'webUI': {
'suites': {
'webUIAddUsers': '',
'webUIAdminSettings': '',
'webUIComments': '',
'webUICreateDelete': '',
'webUIFavorites': '',
'webUIFiles': '',
'webUILogin': '',
'webUIManageQuota': '',
'webUIManageUsersGroups': 'webUIManageUsersGrps',
'webUIMoveFilesFolders': 'webUIMoveFilesFolder',
'webUIPersonalSettings': 'webUIPersonalSetting',
'webUIRenameFiles': '',
'webUIRenameFolders': '',
'webUIRestrictSharing': '',
'webUISettingsMenu': '',
'webUISharingAcceptShares': 'webUISharingAcceptSh',
'webUISharingAutocompletion1': 'webUISharingAutocomp1',
'webUISharingAutocompletion2': 'webUISharingAutocomp2',
'webUISharingInternalGroups1': 'webUISharingIntGroup1',
'webUISharingInternalGroups2': 'webUISharingIntGroup2',
'webUISharingInternalUsers1': 'webUISharingIntUsers1',
'webUISharingInternalUsers2': 'webUISharingIntUsers2',
'webUISharingPublic1': '',
'webUISharingPublic2': '',
'webUITags': '',
'webUITrashbin': '',
'webUIUpload': '',
'webUIWebdavLockProtection': 'webUIWebdavLockProt',
'webUIWebdavLocks': '',
},
'emailNeeded': True,
'useHttps': False,
},
'webUINotifications': {
'suites': {
'webUISharingNotifications': 'webUISharingNotify',
},
'emailNeeded': True,
'useHttps': False,
'extraApps': {
'notifications': 'composer install',
},
},
'webUIFileActionsMenu': {
'suites': {
'webUIFileActionsMenu': '',
},
'useHttps': False,
'extraApps': {
'files_texteditor': 'make vendor',
'richdocuments': 'make vendor',
},
},
'webUIFederation': {
'suites': {
'webUISharingExternal1': 'webUISharingExt1',
'webUISharingExternal2': 'webUISharingExt2',
},
'federatedServerNeeded': True,
'federatedServerVersions': ['git', 'latest', '10.6.0'],
},
'webUIFirefox': {
'suites': {
'webUIFirefoxSmoketest': 'webUIFfSmoke',
},
'browsers': [
'firefox',
],
'emailNeeded': True,
'useHttps': False,
'filterTags': '@smokeTest&&~@notifications-app-required',
'runAllSuites': True,
'numberOfParts': 3,
},
'webUIProxy': {
'suites': {
'webUIProxySmoketest': 'webUIProxySmoke',
},
'browsers': [
'chrome',
],
'emailNeeded': True,
'proxyNeeded': True,
'useHttps': False,
'filterTags': '@smokeTest&&~@notifications-app-required',
'runAllSuites': True,
'numberOfParts': 3,
},
'apiProxy': {
'suites': {
'apiProxySmoketest': 'apiProxySmoke',
},
'proxyNeeded': True,
'useHttps': False,
'filterTags': '@smokeTest&&~@notifications-app-required',
'runAllSuites': True,
'numberOfParts': 8,
},
'apiOnSqlite': {
'suites': {
'apiOnSqlite': 'apiOnSqlite',
},
'databases': ['sqlite'],
'useHttps': False,
'filterTags': '@sqliteDB',
'runAllSuites': True,
}
}
}
def main(ctx):
initial = initialPipelines(ctx)
before = beforePipelines(ctx)
dependsOn(initial, before)
coverageTests = coveragePipelines(ctx)
if (coverageTests == False):
print('Errors detected in coveragePipelines. Review messages above.')
return []
dependsOn(before, coverageTests)
nonCoverageTests = nonCoveragePipelines(ctx)
if (nonCoverageTests == False):
print('Errors detected in nonCoveragePipelines. Review messages above.')
return []
dependsOn(before, nonCoverageTests)
stages = stagePipelines(ctx)
if (stages == False):
print('Errors detected in stagePipelines. Review messages above.')
return []
dependsOn(before, stages)
if (coverageTests == []):
afterCoverageTests = []
else:
afterCoverageTests = afterCoveragePipelines(ctx)
dependsOn(coverageTests, afterCoverageTests)
after = afterPipelines(ctx)
dependsOn(afterCoverageTests + nonCoverageTests + stages, after)
return initial + before + coverageTests + afterCoverageTests + nonCoverageTests + stages + after
def initialPipelines(ctx):
return dependencies(ctx)
def beforePipelines(ctx):
return codestyle() + changelog(ctx) + phpstan() + phan()
def coveragePipelines(ctx):
# All unit test pipelines that have coverage or other test analysis reported
jsPipelines = javascript(ctx, True)
phpUnitPipelines = phpTests(ctx, 'phpunit', True)
phpIntegrationPipelines = phpTests(ctx, 'phpintegration', True)
if (jsPipelines == False) or (phpUnitPipelines == False) or (phpIntegrationPipelines == False):
return False
return jsPipelines + phpUnitPipelines + phpIntegrationPipelines
def nonCoveragePipelines(ctx):
# All unit test pipelines that do not have coverage or other test analysis reported
jsPipelines = javascript(ctx, False)
phpUnitPipelines = phpTests(ctx, 'phpunit', False)
phpIntegrationPipelines = phpTests(ctx, 'phpintegration', False)
if (jsPipelines == False) or (phpUnitPipelines == False) or (phpIntegrationPipelines == False):
return False
return jsPipelines + phpUnitPipelines + phpIntegrationPipelines
def stagePipelines(ctx):
# Pipelines that do not produce coverage or other test analysis reports
litmusPipelines = litmus()
davPipelines = dav()
acceptancePipelines = acceptance(ctx)
if (litmusPipelines == False) or (davPipelines == False) or (acceptancePipelines == False):
return False
return litmusPipelines + davPipelines + acceptancePipelines
def afterCoveragePipelines(ctx):
return [
sonarAnalysis(ctx)
]
def afterPipelines(ctx):
return [
notify()
]
def dependencies(ctx):
pipelines = []
if 'dependencies' not in config:
return pipelines
default = {
'phpVersions': ['7.2'],
}
if 'defaults' in config:
if 'dependencies' in config['defaults']:
for item in config['defaults']['dependencies']:
default[item] = config['defaults']['dependencies'][item]
dependenciesConfig = config['dependencies']
if type(dependenciesConfig) == "bool":
if dependenciesConfig:
# the config has 'dependencies' true, so specify an empty dict that will get the defaults
dependenciesConfig = {}
else:
return pipelines
if len(dependenciesConfig) == 0:
# 'dependencies' is an empty dict, so specify a single section that will get the defaults
dependenciesConfig = {'doDefault': {}}
for category, matrix in dependenciesConfig.items():
params = {}
for item in default:
params[item] = matrix[item] if item in matrix else default[item]
for phpVersion in params['phpVersions']:
name = 'install-dependencies-php%s' % phpVersion
result = {
'kind': 'pipeline',
'type': 'docker',
'name': name,
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'steps':
cacheRestore() +
cacheClearOnEventPush(phpVersion) +
composerInstall(phpVersion) +
vendorbinCodestyle(phpVersion) +
vendorbinCodesniffer(phpVersion) +
vendorbinPhan(phpVersion) +
vendorbinPhpstan(phpVersion) +
vendorbinBehat() +
yarnInstall(phpVersion) +
cacheRebuildOnEventPush() +
cacheFlushOnEventPush(),
'depends_on': [],
'trigger': {
'ref': [
'refs/pull/**',
'refs/tags/**'
]
}
}
for branch in config['branches']:
result['trigger']['ref'].append('refs/heads/%s' % branch)
pipelines.append(result)
return pipelines
def codestyle():
pipelines = []
if 'codestyle' not in config:
return pipelines
default = {
'phpVersions': ['7.4'],
}
if 'defaults' in config:
if 'codestyle' in config['defaults']:
for item in config['defaults']['codestyle']:
default[item] = config['defaults']['codestyle'][item]
codestyleConfig = config['codestyle']
if type(codestyleConfig) == "bool":
if codestyleConfig:
# the config has 'codestyle' true, so specify an empty dict that will get the defaults
codestyleConfig = {}
else:
return pipelines
if len(codestyleConfig) == 0:
# 'codestyle' is an empty dict, so specify a single section that will get the defaults
codestyleConfig = {'doDefault': {}}
for category, matrix in codestyleConfig.items():
params = {}
for item in default:
params[item] = matrix[item] if item in matrix else default[item]
for phpVersion in params['phpVersions']:
name = 'coding-standard-php%s' % phpVersion
result = {
'kind': 'pipeline',
'type': 'docker',
'name': name,
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'steps':
cacheRestore() +
composerInstall(phpVersion) +
vendorbinCodestyle(phpVersion) +
vendorbinCodesniffer(phpVersion) +
yarnInstall(phpVersion) +
[
{
'name': 'php-style',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'make test-php-style'
]
}
],
'depends_on': [],
'trigger': {
'ref': [
'refs/pull/**',
'refs/tags/**'
]
}
}
for branch in config['branches']:
result['trigger']['ref'].append('refs/heads/%s' % branch)
pipelines.append(result)
return pipelines
def changelog(ctx):
repo_slug = ctx.build.source_repo if ctx.build.source_repo else ctx.repo.slug
pipelines = []
result = {
'kind': 'pipeline',
'type': 'docker',
'name': 'changelog',
'clone': {
'disable': True,
},
'steps':
[
{
'name': 'clone',
'image': 'plugins/git-action:1',
'pull': 'always',
'settings': {
'actions': [
'clone',
],
'remote': 'https://github.com/%s' % (repo_slug),
'branch': ctx.build.source if ctx.build.event == 'pull_request' else 'master',
'path': dir['server'],
'netrc_machine': 'github.com',
'netrc_username': {
'from_secret': 'github_username',
},
'netrc_password': {
'from_secret': 'github_token',
},
},
},
{
'name': 'generate',
'image': 'toolhippie/calens:latest',
'pull': 'always',
'commands': [
'calens >| CHANGELOG.md',
'calens -t changelog/CHANGELOG-html.tmpl >| CHANGELOG.html',
],
},
{
'name': 'diff',
'image': 'owncloudci/alpine:latest',
'pull': 'always',
'commands': [
'git diff',
],
},
{
'name': 'output',
'image': 'toolhippie/calens:latest',
'pull': 'always',
'commands': [
'cat CHANGELOG.md',
],
},
{
'name': 'publish',
'image': 'plugins/git-action:1',
'pull': 'always',
'settings': {
'actions': [
'commit',
'push',
],
'message': 'Automated changelog update [skip ci]',
'branch': 'master',
'author_email': 'devops@owncloud.com',
'author_name': 'ownClouders',
'netrc_machine': 'github.com',
'netrc_username': {
'from_secret': 'github_username',
},
'netrc_password': {
'from_secret': 'github_token',
},
},
'when': {
'ref': {
'exclude': [
'refs/pull/**',
],
},
},
},
],
'depends_on': [],
'trigger': {
'ref': [
'refs/heads/master',
'refs/pull/**',
],
},
}
pipelines.append(result)
return pipelines
def phpstan():
pipelines = []
if 'phpstan' not in config:
return pipelines
default = {
'phpVersions': ['7.4'],
'logLevel': '2',
}
if 'defaults' in config:
if 'phpstan' in config['defaults']:
for item in config['defaults']['phpstan']:
default[item] = config['defaults']['phpstan'][item]
phpstanConfig = config['phpstan']
if type(phpstanConfig) == "bool":
if phpstanConfig:
# the config has 'phpstan' true, so specify an empty dict that will get the defaults
phpstanConfig = {}
else:
return pipelines
if len(phpstanConfig) == 0:
# 'phpstan' is an empty dict, so specify a single section that will get the defaults
phpstanConfig = {'doDefault': {}}
for category, matrix in phpstanConfig.items():
params = {}
for item in default:
params[item] = matrix[item] if item in matrix else default[item]
for phpVersion in params['phpVersions']:
name = 'phpstan-php%s' % phpVersion
result = {
'kind': 'pipeline',
'type': 'docker',
'name': name,
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'steps':
cacheRestore() +
composerInstall(phpVersion) +
vendorbinPhpstan(phpVersion) +
yarnInstall(phpVersion) +
installServer(phpVersion, 'sqlite', params['logLevel']) +
[
{
'name': 'php-phpstan',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'make test-php-phpstan'
]
}
],
'depends_on': [],
'trigger': {
'ref': [
'refs/pull/**',
'refs/tags/**'
]
}
}
pipelines.append(result)
return pipelines
def phan():
pipelines = []
if 'phan' not in config:
return pipelines
default = {
'phpVersions': ['7.2', '7.3', '7.4'],
'logLevel': '2',
}
if 'defaults' in config:
if 'phan' in config['defaults']:
for item in config['defaults']['phan']:
default[item] = config['defaults']['phan'][item]
phanConfig = config['phan']
if type(phanConfig) == "bool":
if phanConfig:
# the config has 'phan' true, so specify an empty dict that will get the defaults
phanConfig = {}
else:
return pipelines
if len(phanConfig) == 0:
# 'phan' is an empty dict, so specify a single section that will get the defaults
phanConfig = {'doDefault': {}}
for category, matrix in phanConfig.items():
params = {}
for item in default:
params[item] = matrix[item] if item in matrix else default[item]
for phpVersion in params['phpVersions']:
name = 'phan-php%s' % phpVersion
result = {
'kind': 'pipeline',
'type': 'docker',
'name': name,
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'steps':
cacheRestore() +
composerInstall(phpVersion) +
vendorbinPhan(phpVersion) +
yarnInstall(phpVersion) +
installServer(phpVersion, 'sqlite', params['logLevel']) +
[
{
'name': 'phan',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'make test-php-phan'
]
}
],
'depends_on': [],
'trigger': {
'ref': [
'refs/pull/**',
'refs/tags/**'
]
}
}
pipelines.append(result)
return pipelines
def litmus():
pipelines = []
if 'litmus' not in config:
return pipelines
default = {
'phpVersions': ['7.2', '7.3', '7.4'],
'logLevel': '2',
'useHttps': True,
}
if 'defaults' in config:
if 'litmus' in config['defaults']:
for item in config['defaults']['litmus']:
default[item] = config['defaults']['litmus'][item]
litmusConfig = config['litmus']
if type(litmusConfig) == "bool":
if litmusConfig:
# the config has 'litmus' true, so specify an empty dict that will get the defaults
litmusConfig = {}
else:
return pipelines
if len(litmusConfig) == 0:
# 'litmus' is an empty dict, so specify a single section that will get the defaults
litmusConfig = {'doDefault': {}}
for category, matrix in litmusConfig.items():
params = {}
for item in default:
params[item] = matrix[item] if item in matrix else default[item]
for phpVersion in params['phpVersions']:
name = 'litmus-php%s' % phpVersion
db = 'mariadb:10.2'
image = 'owncloud/litmus:latest'
environment = {
'LITMUS_PASSWORD': 'admin',
'LITMUS_USERNAME': 'admin',
'TESTS': 'basic copymove props locks http',
}
litmusCommand = '/usr/local/bin/litmus-wrapper'
result = {
'kind': 'pipeline',
'type': 'docker',
'name': name,
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'steps':
cacheRestore() +
composerInstall(phpVersion) +
yarnInstall(phpVersion) +
installServer(phpVersion, db, params['logLevel'], params['useHttps']) +
setupLocalStorage(phpVersion) +
fixPermissions(phpVersion, False) +
createShare(phpVersion) +
owncloudLog('server', 'src') +
[
{
'name': 'old-endpoint',
'image': image,
'pull': 'always',
'environment': environment,
'commands': [
'source .env',
'export LITMUS_URL="https://server/remote.php/webdav"',
litmusCommand,
]
},
{
'name': 'new-endpoint',
'image': image,
'pull': 'always',
'environment': environment,
'commands': [
'source .env',
'export LITMUS_URL="https://server/remote.php/dav/files/admin"',
litmusCommand,
]
},
{
'name': 'new-mount',
'image': image,
'pull': 'always',
'environment': environment,
'commands': [
'source .env',
'export LITMUS_URL="https://server/remote.php/dav/files/admin/local_storage/"',
litmusCommand,
]
},
{
'name': 'old-mount',
'image': image,
'pull': 'always',
'environment': environment,
'commands': [
'source .env',
'export LITMUS_URL="https://server/remote.php/webdav/local_storage/"',
litmusCommand,
]
},
{
'name': 'new-shared',
'image': image,
'pull': 'always',
'environment': environment,
'commands': [
'source .env',
'export LITMUS_URL="https://server/remote.php/dav/files/admin/new_folder/"',
litmusCommand,
]
},
{
'name': 'old-shared',
'image': image,
'pull': 'always',
'environment': environment,
'commands': [
'source .env',
'export LITMUS_URL="https://server/remote.php/webdav/new_folder/"',
litmusCommand,
]
},
{
'name': 'public-share',
'image': image,
'pull': 'always',
'environment': {
'LITMUS_PASSWORD': 'admin',
'LITMUS_USERNAME': 'admin',
'TESTS': 'basic copymove http',
},
'commands': [
'source .env',
'export LITMUS_URL=\'https://server/remote.php/dav/public-files/\'$PUBLIC_TOKEN',
litmusCommand,
]
},
],
'services':
databaseService(db) +
owncloudService(phpVersion, 'server', dir['server'], params['useHttps']),
'depends_on': [],
'trigger': {
'ref': [
'refs/pull/**',
'refs/tags/**'
]
}
}
pipelines.append(result)
return pipelines
def dav():
pipelines = []
if 'dav' not in config:
return pipelines
default = {
'phpVersions': ['7.2', '7.3', '7.4'],
'logLevel': '2'
}
if 'defaults' in config:
if 'dav' in config['defaults']:
for item in config['defaults']['dav']:
default[item] = config['defaults']['dav'][item]
davConfig = config['dav']
if type(davConfig) == "bool":
if davConfig:
# the config has 'dav' true, so specify an empty dict that will get the defaults
davConfig = {}
else:
return pipelines
if len(davConfig) == 0:
# 'dav' is an empty dict, so specify a single section that will get the defaults
davConfig = {'doDefault': {}}
for category, matrix in davConfig.items():
params = {}
for item in default:
params[item] = matrix[item] if item in matrix else default[item]
for phpVersion in params['phpVersions']:
for davType in ['caldav-new', 'caldav-old', 'carddav-new', 'carddav-old']:
name = '%s-php%s' % (davType, phpVersion)
db = 'mariadb:10.2'
if (davType == 'caldav-new'):
scriptPath = 'apps/dav/tests/ci/caldav'
if (davType == 'caldav-old'):
scriptPath = 'apps/dav/tests/ci/caldav-old-endpoint'
if (davType == 'carddav-new'):
scriptPath = 'apps/dav/tests/ci/carddav'
if (davType == 'carddav-old'):
scriptPath = 'apps/dav/tests/ci/carddav-old-endpoint'
result = {
'kind': 'pipeline',
'type': 'docker',
'name': name,
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'steps':
cacheRestore() +
composerInstall(phpVersion) +
yarnInstall(phpVersion) +
installServer(phpVersion, db, params['logLevel']) +
davInstall(phpVersion, scriptPath) +
fixPermissions(phpVersion, False) +
owncloudLog('server', 'src') +
[
{
'name': 'dav-test',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'bash %s/script.sh' % scriptPath,
]
},
],
'services':
databaseService(db),
'depends_on': [],
'trigger': {
'ref': [
'refs/pull/**',
'refs/tags/**'
]
}
}
pipelines.append(result)
return pipelines
def javascript(ctx, withCoverage):
pipelines = []
if 'javascript' not in config:
return pipelines
default = {
'coverage': True,
'logLevel': '2',
'phpVersion': '7.2',
'skip': False
}
if 'defaults' in config:
if 'javascript' in config['defaults']:
for item in config['defaults']['javascript']:
default[item] = config['defaults']['javascript'][item]
matrix = config['javascript']
if type(matrix) == "bool":
if matrix:
# the config has 'javascript' true, so specify an empty dict that will get the defaults
matrix = {}
else:
return pipelines
params = {}
for item in default:
params[item] = matrix[item] if item in matrix else default[item]
if params['skip']:
return pipelines
# if we only want pipelines with coverage, and this pipeline does not do coverage, then do not include it
if withCoverage and not params['coverage']:
return pipelines
# if we only want pipelines without coverage, and this pipeline does coverage, then do not include it
if not withCoverage and params['coverage']:
return pipelines
result = {
'kind': 'pipeline',
'type': 'docker',
'name': 'test-javascript',
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'steps':
cacheRestore() +
composerInstall(params['phpVersion']) +
yarnInstall(params['phpVersion']) +
[
{
'name': 'test-js',
'image': 'owncloudci/php:%s' % params['phpVersion'],
'pull': 'always',
'commands': [
'make test-js'
]
}
],
'depends_on': [],
'trigger': {
'ref': [
'refs/pull/**',
'refs/tags/**'
]
}
}
if params['coverage']:
result['steps'].append(
{
'name': 'coverage-cache',
'image': 'plugins/s3',
'pull': 'always',
'settings': {
'endpoint': {
'from_secret': 'cache_s3_endpoint'
},
'bucket': 'cache',
'source': 'tests/output/coverage/PhantomJS 2.1.1 (Linux 0.0.0)/lcov.info',
'target': '%s/%s/coverage' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'),
'path_style': True,
'strip_prefix': 'tests/output/coverage/PhantomJS 2.1.1 (Linux 0.0.0)',
'access_key': {
'from_secret': 'cache_s3_access_key'
},
'secret_key': {
'from_secret': 'cache_s3_secret_key'
}
}
}
)
for branch in config['branches']:
result['trigger']['ref'].append('refs/heads/%s' % branch)
return [result]
def phpTests(ctx, testType, withCoverage):
pipelines = []
if testType not in config:
return pipelines
errorFound = False
default = {
'phpVersions': ['7.2', '7.3', '7.4'],
'databases': [
'sqlite',
'mariadb:10.2',
'mariadb:10.3',
'mariadb:10.4',
'mariadb:10.5',
'mysql:5.5',
'mysql:5.7',
'mysql:8.0',
'postgres:9.4',
'postgres:10.3',
'oracle'
],
'coverage': True,
'includeKeyInMatrixName': False,
'logLevel': '2',
'cephS3': False,
'scalityS3': False,
'externalTypes': ['none'],
'extraSetup': [],
'extraServices': [],
'extraEnvironment': {},
'extraCommandsBeforeTestRun': [],
'extraApps': {},
'skip': False
}
if 'defaults' in config:
if testType in config['defaults']:
for item in config['defaults'][testType]:
default[item] = config['defaults'][testType][item]
phpTestConfig = config[testType]
if type(phpTestConfig) == "bool":
if phpTestConfig:
# the config has just True, so specify an empty dict that will get the defaults
phpTestConfig = {}
else:
return pipelines
if len(phpTestConfig) == 0:
# the PHP test config is an empty dict, so specify a single section that will get the defaults
phpTestConfig = {'doDefault': {}}
for category, matrix in phpTestConfig.items():
params = {}
for item in default:
params[item] = matrix[item] if item in matrix else default[item]
if params['skip']:
continue
# if we only want pipelines with coverage, and this pipeline does not do coverage, then do not include it
if withCoverage and not params['coverage']:
continue
# if we only want pipelines without coverage, and this pipeline does coverage, then do not include it
if not withCoverage and params['coverage']:
continue
for phpVersion in params['phpVersions']:
if testType == 'phpunit':
command = 'su-exec www-data bash tests/drone/test-phpunit.sh'
else:
command = 'unknown tbd'
for db in params['databases']:
for externalType in params['externalTypes']:
keyString = '-' + category if params['includeKeyInMatrixName'] else ''
filesExternalType = externalType if externalType != 'none' else ''
externalNameString = '-' + externalType if externalType != 'none' else ''
name = '%s%s-php%s-%s%s' % (testType, keyString, phpVersion, getShortDbNameAndVersion(db), externalNameString)
maxLength = 50
nameLength = len(name)
if nameLength > maxLength:
print("Error: generated phpunit stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name)
errorFound = True
if ((externalType == 'scality') or (params['scalityS3'] != False)):
needScality = True
filesExternalType = ''
primaryObjectStore = 'files_primary_s3'
else:
needScality = False
primaryObjectStore = ''
if (filesExternalType == ''):
# for the regular unit test runs, the clover coverage results are in a file named like:
# autotest-clover-sqlite.xml
coverageFileNameStart = 'autotest'
extraCoverageRenameCommand = []
extraCoverage = False
else:
# for the files-external unit test runs, the clover coverage results are in 2 files named like:
# autotest-external-clover-sqlite.xml
# autotest-external-clover-sqlite-samba.xml
coverageFileNameStart = 'autotest-external'
extraCoverageRenameCommand = [
'mv tests/output/coverage/%s-clover-%s-%s.xml tests/output/coverage/clover-%s-%s.xml' % (coverageFileNameStart, getDbName(db), externalType, name, externalType)
]
extraCoverage = True
if ((externalType == 'scality') or (params['cephS3'] != False) or (params['scalityS3'] != False)):
# If we need S3 storage, then install the 'files_primary_s3' app
extraAppsDict = {
'files_primary_s3': 'composer install'
}
else:
extraAppsDict = {}
if (externalType == 'owncloud'):
needRedis = True
else:
needRedis = False
for app, command in params['extraApps'].items():
extraAppsDict[app] = command
result = {
'kind': 'pipeline',
'type': 'docker',
'name': name,
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'steps':
cacheRestore() +
composerInstall(phpVersion) +
yarnInstall(phpVersion) +
installServer(phpVersion, db, params['logLevel']) +
installExtraApps(phpVersion, extraAppsDict, dir['server']) +
setupScality(phpVersion, needScality) +
params['extraSetup'] +
fixPermissions(phpVersion, False) +
owncloudLog('server', 'src') +
[
{
'name': '%s-tests' % testType,
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': {
'COVERAGE': params['coverage'],
'DB_TYPE': getDbName(db),
'FILES_EXTERNAL_TYPE': filesExternalType,
'PRIMARY_OBJECTSTORE': primaryObjectStore
},
'commands': params['extraCommandsBeforeTestRun'] + [
command
]
}
],
'services':
databaseService(db) +
cephService(params['cephS3']) +
scalityService(needScality) +
webdavService(externalType == 'webdav') +
sambaService(externalType == 'samba') +
sftpService(externalType == 'sftp') +
redisService(needRedis) +
owncloudDockerService(externalType == 'owncloud') +
params['extraServices'],
'depends_on': [],
'trigger': {
'ref': [
'refs/pull/**',
'refs/tags/**'
]
}
}
if params['coverage']:
result['steps'].append({
'name': 'coverage-rename',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'mv tests/output/coverage/%s-clover-%s.xml tests/output/coverage/clover-%s.xml' % (coverageFileNameStart, getDbName(db), name)
] + extraCoverageRenameCommand
})
result['steps'].append({
'name': 'coverage-cache-1',
'image': 'plugins/s3',
'pull': 'always',
'settings': {
'endpoint': {
'from_secret': 'cache_s3_endpoint'
},
'bucket': 'cache',
'source': 'tests/output/coverage/clover-%s.xml' % (name),
'target': '%s/%s/coverage' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'),
'path_style': True,
'strip_prefix': 'tests/output/coverage',
'access_key': {
'from_secret': 'cache_s3_access_key'
},
'secret_key': {
'from_secret': 'cache_s3_secret_key'
}
}
})
if extraCoverage:
result['steps'].append({
'name': 'coverage-cache-2',
'image': 'plugins/s3',
'pull': 'always',
'settings': {
'endpoint': {
'from_secret': 'cache_s3_endpoint'
},
'bucket': 'cache',
'source': 'tests/output/coverage/clover-%s-%s.xml' % (name, externalType),
'target': '%s/%s/coverage' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'),
'path_style': True,
'strip_prefix': 'tests/output/coverage',
'access_key': {
'from_secret': 'cache_s3_access_key'
},
'secret_key': {
'from_secret': 'cache_s3_secret_key'
}
}
})
for branch in config['branches']:
result['trigger']['ref'].append('refs/heads/%s' % branch)
pipelines.append(result)
if errorFound:
return False
return pipelines
def acceptance(ctx):
pipelines = []
if 'acceptance' not in config:
return pipelines
if type(config['acceptance']) == "bool":
if not config['acceptance']:
return pipelines
errorFound = False
default = {
'federatedServerVersions': [''],
'browsers': ['chrome'],
'phpVersions': ['7.4'],
'databases': ['mariadb:10.2'],
'federatedPhpVersion': '7.2',
'federatedServerNeeded': False,
'federatedDb': '',
'filterTags': '',
'logLevel': '2',
'emailNeeded': False,
'ldapNeeded': False,
'proxyNeeded': False,
'cephS3': False,
'scalityS3': False,
'testingRemoteSystem': True,
'useHttps': True,
'replaceUsernames': False,
'extraSetup': [],
'extraServices': [],
'extraEnvironment': {'OC_LANGUAGE':'en-EN'},
'extraCommandsBeforeTestRun': [],
'extraApps': {},
'useBundledApp': False,
'includeKeyInMatrixName': False,
'runAllSuites': False,
'numberOfParts': 1,
'skip': False,
'debugSuites': [],
'skipExceptParts': [],
'testAgainstCoreTarball': False,
'coreTarball': 'daily-master-qa',
'earlyFail': True,
}
if 'defaults' in config:
if 'acceptance' in config['defaults']:
for item in config['defaults']['acceptance']:
default[item] = config['defaults']['acceptance'][item]
for category, matrix in config['acceptance'].items():
if type(matrix['suites']) == "list":
suites = {}
for suite in matrix['suites']:
suites[suite] = suite
else:
suites = matrix['suites']
if 'debugSuites' in matrix and len(matrix['debugSuites']) != 0:
if type(matrix['debugSuites']) == "list":
suites = {}
for suite in matrix['debugSuites']:
suites[suite] = suite
else:
suites = matrix['debugSuites']
for suite, alternateSuiteName in suites.items():
isWebUI = suite.startswith('webUI')
isAPI = suite.startswith('api')
isCLI = suite.startswith('cli')
if (alternateSuiteName == ''):
alternateSuiteName = suite
params = {}
for item in default:
params[item] = matrix[item] if item in matrix else default[item]
if params['skip']:
continue
if ("full-ci" in ctx.build.title.lower()):
params["earlyFail"] = False
if isAPI or isCLI:
params['browsers'] = ['']
needObjectStore = (params['cephS3'] != False) or (params['scalityS3'] != False)
extraAppsDict = {}
if not params['testAgainstCoreTarball']:
extraAppsDict['testing'] = 'composer install'
if (needObjectStore):
# If we need S3 object storage, then install the 'files_primary_s3' app
extraAppsDict['files_primary_s3'] = 'composer install'
for app, command in params['extraApps'].items():
extraAppsDict[app] = command
for federatedServerVersion in params['federatedServerVersions']:
for browser in params['browsers']:
for phpVersion in params['phpVersions']:
for db in params['databases']:
for runPart in range(1, params['numberOfParts'] + 1):
debugPartsEnabled = (len(params['skipExceptParts']) != 0)
if debugPartsEnabled and runPart not in params['skipExceptParts']:
continue
name = 'unknown'
federatedDb = db if params['federatedDb'] == '' else params['federatedDb']
federatedDbName = getDbName(federatedDb)
if federatedDbName not in ['mariadb', 'mysql']:
# Do not try to run 2 sets of Oracle, Postgres etc databases
# When testing with these, let the federated server use mariadb
federatedDb = 'mariadb:10.2'
if isWebUI or isAPI or isCLI:
browserString = '' if browser == '' else '-' + browser
keyString = '-' + category if params['includeKeyInMatrixName'] else ''
partString = '' if params['numberOfParts'] == 1 else '-%d-%d' % (params['numberOfParts'], runPart)
federatedServerVersionString = '-' + federatedServerVersion.replace('daily-', '').replace('-qa', '') if (federatedServerVersion != '') else ''
name = '%s%s%s%s%s-%s-php%s' % (alternateSuiteName, keyString, partString, federatedServerVersionString, browserString, getShortDbNameAndVersion(db), phpVersion)
maxLength = 50
nameLength = len(name)
if nameLength > maxLength:
print("Error: generated stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name)
errorFound = True
environment = {}
for env in params['extraEnvironment']:
environment[env] = params['extraEnvironment'][env]
if (params['useHttps']):
protocol = 'https'
else:
protocol = 'http'
if (params['proxyNeeded']):
serverUnderTest = 'proxy'
else:
serverUnderTest = 'server'
environment['TEST_SERVER_URL'] = '%s://%s' % (protocol, serverUnderTest)
environment['BEHAT_FILTER_TAGS'] = params['filterTags']
environment['REPLACE_USERNAMES'] = params['replaceUsernames']
if (params['runAllSuites'] == False):
environment['BEHAT_SUITE'] = suite
else:
environment['DIVIDE_INTO_NUM_PARTS'] = params['numberOfParts']
environment['RUN_PART'] = runPart
if isWebUI:
environment['SELENIUM_HOST'] = 'selenium'
environment['SELENIUM_PORT'] = '4444'
environment['BROWSER'] = browser
environment['PLATFORM'] = 'Linux'
makeParameter = 'test-acceptance-webui'
if isAPI:
makeParameter = 'test-acceptance-api'
if isCLI:
makeParameter = 'test-acceptance-cli'
if params['emailNeeded']:
environment['MAILHOG_HOST'] = 'email'
if params['ldapNeeded']:
environment['TEST_WITH_LDAP'] = True
if params['testingRemoteSystem']:
environment['TESTING_REMOTE_SYSTEM'] = True
suExecCommand = ''
else:
environment['TESTING_REMOTE_SYSTEM'] = False
# The test suite (may/will) run local commands, rather than calling the testing app to do them
# Those commands need to be executed as www-data (which owns the files)
suExecCommand = 'su-exec www-data '
if params['testAgainstCoreTarball']:
pathOfServerUnderTest = '/drone/core'
else:
pathOfServerUnderTest = dir['server']
if (needObjectStore):
environment['OC_TEST_ON_OBJECTSTORE'] = '1'
if (params['cephS3'] != False):
environment['S3_TYPE'] = 'ceph'
if (params['scalityS3'] != False):
environment['S3_TYPE'] = 'scality'
federationDbSuffix = 'fed'
result = {
'kind': 'pipeline',
'type': 'docker',
'name': name,
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'steps':
cacheRestore() +
composerInstall(phpVersion) +
vendorbinBehat() +
yarnInstall(phpVersion) +
((
installCoreFromTarball(params['coreTarball'], db, params['logLevel'], params['useHttps'], params['federatedServerNeeded'], params['proxyNeeded'], pathOfServerUnderTest)
) if params['testAgainstCoreTarball'] else (
installServer(phpVersion, db, params['logLevel'], params['useHttps'], params['federatedServerNeeded'], params['proxyNeeded'])
)) +
(
installAndConfigureFederated(ctx, federatedServerVersion, params['federatedPhpVersion'], params['logLevel'], protocol, federatedDb, federationDbSuffix) +
owncloudLog('federated', 'federated') if params['federatedServerNeeded'] else []
) +
installExtraApps(phpVersion, extraAppsDict, pathOfServerUnderTest) +
setupCeph(phpVersion, params['cephS3']) +
setupScality(phpVersion, params['scalityS3']) +
params['extraSetup'] +
fixPermissions(phpVersion, params['federatedServerNeeded'], pathOfServerUnderTest) +
waitForServer(phpVersion, params['federatedServerNeeded']) +
owncloudLog('server', pathOfServerUnderTest) +
[
({
'name': 'acceptance-tests',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': environment,
'commands': params['extraCommandsBeforeTestRun'] + [
'touch %s/saved-settings.sh' % dir['base'],
'. %s/saved-settings.sh' % dir['base'],
'%smake %s' % (suExecCommand, makeParameter)
]
}),
] + buildGithubCommentForBuildStopped(name, params['earlyFail']) + githubComment(params['earlyFail']) + stopBuild(params['earlyFail']) ,
'services':
databaseService(db) +
browserService(browser) +
emailService(params['emailNeeded']) +
ldapService(params['ldapNeeded']) +
proxyService(params['proxyNeeded']) +
cephService(params['cephS3']) +
scalityService(params['scalityS3']) +
params['extraServices'] +
owncloudService(phpVersion, 'server', pathOfServerUnderTest, params['useHttps']) +
((
owncloudService(params['federatedPhpVersion'], 'federated', dir["federated"], params['useHttps']) +
databaseServiceForFederation(federatedDb, federationDbSuffix)
) if params['federatedServerNeeded'] else []),
'depends_on': [],
'trigger': {
'ref': [
'refs/pull/**',
'refs/tags/**'
]
}
}
pipelines.append(result)
if errorFound:
return False
return pipelines
def sonarAnalysis(ctx, phpVersion = '7.4'):
sonar_env = {
"SONAR_TOKEN": {
"from_secret": "sonar_token",
},
'SONAR_SCANNER_OPTS': '-Xdebug'
}
if ctx.build.event == "pull_request":
sonar_env.update({
"SONAR_PULL_REQUEST_BASE": "%s" % (ctx.build.target),
"SONAR_PULL_REQUEST_BRANCH": "%s" % (ctx.build.source),
"SONAR_PULL_REQUEST_KEY": "%s" % (ctx.build.ref.replace("refs/pull/", "").split("/")[0]),
})
repo_slug = ctx.build.source_repo if ctx.build.source_repo else ctx.repo.slug
result = {
'kind': 'pipeline',
'type': 'docker',
'name': 'sonar-analysis',
'workspace' : {
'base': dir['base'],
'path': 'src'
},
'clone': {
'disable': True, # Sonarcloud does not apply issues on already merged branch
},
'steps': [
{
"name": "clone",
"image": "owncloudci/alpine:latest",
"commands": [
"git clone https://github.com/%s.git ." % (repo_slug),
"git checkout $DRONE_COMMIT",
],
},
] +
cacheRestore() +
composerInstall(phpVersion) +
yarnInstall(phpVersion) +
installServer(phpVersion, 'sqlite') +
[
{
'name': 'sync-from-cache',
'image': 'minio/mc:RELEASE.2020-12-10T01-26-17Z',
'pull': 'always',
'environment': {
'MC_HOST_cache': {
'from_secret': 'cache_s3_connection_url'
},
},
'commands': [
'mkdir -p results',
'mc mirror cache/cache/%s/%s/coverage results/' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'),
]
},
{
'name': 'setup-before-sonarcloud',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'pwd',
'ls -l',
'ls -l results',
'ls -l apps',
'ls -l config',
'cd apps',
'git clone https://github.com/owncloud/files_primary_s3.git',
'cd files_primary_s3',
'composer install',
'cd %s' % dir['server']
]
},
{
'name': 'sonarcloud',
'image': 'sonarsource/sonar-scanner-cli',
'pull': 'always',
'environment': sonar_env,
},
{
'name': 'purge-cache',
'image': 'minio/mc:RELEASE.2020-12-10T01-26-17Z',
'environment': {
'MC_HOST_cache': {
'from_secret': 'cache_s3_connection_url'
}
},
'commands': [
'mc rm --recursive --force cache/cache/%s/%s' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}'),
]
},
],
'depends_on': [],
'trigger': {
'ref': [
'refs/heads/master',
'refs/pull/**',
'refs/tags/**'
]
}
}
for branch in config['branches']:
result['trigger']['ref'].append('refs/heads/%s' % branch)
return result
def notify():
result = {
'kind': 'pipeline',
'type': 'docker',
'name': 'chat-notifications',
'clone': {
'disable': True
},
'steps': [
{
'name': 'notify-rocketchat',
'image': 'plugins/slack:1',
'pull': 'always',
'settings': {
'webhook': {
'from_secret': config['rocketchat']['from_secret']
},
'channel': config['rocketchat']['channel']
}
}
],
'depends_on': [],
'trigger': {
'ref': [
'refs/tags/**'
],
'status': [
'success',
'failure'
]
}
}
for branch in config['branches']:
result['trigger']['ref'].append('refs/heads/%s' % branch)
return result
def stopBuild(earlyFail):
if (earlyFail):
return [{
"name": "stop-build",
"image": "drone/cli:alpine",
"pull": "always",
"environment": {
"DRONE_SERVER": "https://drone.owncloud.com",
"DRONE_TOKEN": {
"from_secret": "drone_token",
},
},
"commands": [
"drone build stop owncloud/core ${DRONE_BUILD_NUMBER}",
],
"when": {
"status": [
"failure",
],
"event": [
"pull_request",
],
},
}]
else:
return []
def buildGithubCommentForBuildStopped(alternateSuiteName, earlyFail):
if (earlyFail):
return [{
"name": "build-github-comment-buildStop",
"image": "owncloud/ubuntu:16.04",
"pull": "always",
"commands": [
'echo ":boom: Acceptance tests pipeline <strong>%s</strong> failed. The build has been cancelled.\\n\\n${DRONE_BUILD_LINK}/${DRONE_JOB_NUMBER}${DRONE_STAGE_NUMBER}/1\\n" >> %s/comments.file' % (alternateSuiteName, dir['server']),
],
"when": {
"status": [
"failure",
],
"event": [
"pull_request",
],
},
}]
else:
return []
def githubComment(earlyFail):
if (earlyFail):
return [{
"name": "github-comment",
"image": "jmccann/drone-github-comment:1",
"pull": "if-not-exists",
"settings": {
"message_file": "%s/comments.file" % dir['server'],
},
"environment": {
"GITHUB_TOKEN": {
"from_secret": "github_token",
},
},
"when": {
"status": [
"failure",
],
"event": [
"pull_request",
],
},
}]
else:
return []
def databaseService(db):
dbName = getDbName(db)
if (dbName == 'mariadb') or (dbName == 'mysql'):
service = {
'name': dbName,
'image': db,
'pull': 'always',
'environment': {
'MYSQL_USER': getDbUsername(db),
'MYSQL_PASSWORD': getDbPassword(db),
'MYSQL_DATABASE': getDbDatabase(db),
'MYSQL_ROOT_PASSWORD': getDbRootPassword()
}
}
if (db == 'mysql:8.0'):
service['command'] = ['--default-authentication-plugin=mysql_native_password']
return [service]
if dbName == 'postgres':
return [{
'name': dbName,
'image': db,
'pull': 'always',
'environment': {
'POSTGRES_USER': getDbUsername(db),
'POSTGRES_PASSWORD': getDbPassword(db),
'POSTGRES_DB': getDbDatabase(db)
}
}]
if dbName == 'oracle':
return [{
'name': dbName,
'image': 'owncloudci/oracle-xe:latest',
'pull': 'always',
'environment': {
'ORACLE_USER': getDbUsername(db),
'ORACLE_PASSWORD': getDbPassword(db),
'ORACLE_DB': getDbDatabase(db),
'ORACLE_DISABLE_ASYNCH_IO': 'true',
}
}]
return []
def browserService(browser):
if browser == 'chrome':
return [{
'name': 'selenium',
'image': 'selenium/standalone-chrome-debug:3.141.59-oxygen',
'pull': 'always',
'environment': {
'JAVA_OPTS': '-Dselenium.LOGGER.level=WARNING'
}
}]
if browser == 'firefox':
return [{
'name': 'selenium',
'image': 'selenium/standalone-firefox-debug:3.8.1',
'pull': 'always',
'environment': {
'JAVA_OPTS': '-Dselenium.LOGGER.level=WARNING',
'SE_OPTS': '-enablePassThrough false'
}
}]
return []
def emailService(emailNeeded):
if emailNeeded:
return [{
'name': 'email',
'image': 'mailhog/mailhog',
'pull': 'always',
}]
return []
def ldapService(ldapNeeded):
if ldapNeeded:
return [{
'name': 'ldap',
'image': 'osixia/openldap',
'pull': 'always',
'environment': {
'LDAP_DOMAIN': 'owncloud.com',
'LDAP_ORGANISATION': 'owncloud',
'LDAP_ADMIN_PASSWORD': 'admin',
'LDAP_TLS_VERIFY_CLIENT': 'never',
}
}]
return []
def proxyService(proxyNeeded):
if proxyNeeded:
return [{
'name': 'proxy',
'image': 'pottava/proxy',
'pull': 'always',
'environment': {
'PROXY_URL': 'http://server',
}
}]
return []
def webdavService(needed):
if not needed:
return []
return [{
'name': 'webdav',
'image': 'owncloudci/php:latest',
'pull': 'always',
'environment': {
'APACHE_CONFIG_TEMPLATE': 'webdav'
},
'command': [
'/usr/local/bin/apachectl',
'-D',
'FOREGROUND'
]
}]
def sambaService(needed):
if not needed:
return []
return [{
'name': 'samba',
'image': 'owncloudci/samba:latest',
'pull': 'always',
'command': [
'-u',
'test;test',
'-s',
'public;/tmp;yes;no;no;test;none;test',
'-S',
]
}]
def sftpService(needed):
if not needed:
return []
# 'test:12345:1001::upload'
# Creates a user 'test' with password '12345', UID '1001'
# The user has a folder called 'upload' that they can write to.
# The tests depend on the 'upload' folder already existing, because
# "Just remember that the users can't create new files directly under their
# own home directory, so make sure there are at least one subdirectory
# if you want them to upload files" https://hub.docker.com/r/atmoz/sftp/
# Also see apps/files_external/tests/env/start-sftp-atmoz.sh where the
# docker image can be started "locally".
return [{
'name': 'sftp',
'image': 'atmoz/sftp',
'pull': 'always',
'environment': {
'SFTP_USERS': 'test:12345:1001::upload'
}
}]
def scalityService(scalityS3):
if not scalityS3:
return []
return [{
'name': 'scality',
'image': 'owncloudci/scality-s3server:latest',
'pull': 'always',
'environment': {
'HOST_NAME': 'scality'
}
}]
def cephService(cephS3):
if not cephS3:
return []
return [{
'name': 'ceph',
'image': 'owncloudci/ceph:tag-build-master-jewel-ubuntu-16.04',
'pull': 'always',
'environment': {
'NETWORK_AUTO_DETECT': '4',
'RGW_NAME': 'ceph',
'CEPH_DEMO_UID': 'owncloud',
'CEPH_DEMO_ACCESS_KEY': 'owncloud123456',
'CEPH_DEMO_SECRET_KEY': 'secret123456',
}
}]
def owncloudService(phpVersion, name, pathOfServerUnderTest, ssl):
if ssl:
environment = {
'APACHE_WEBROOT': pathOfServerUnderTest,
'APACHE_CONFIG_TEMPLATE': 'ssl',
'APACHE_SSL_CERT_CN': name,
'APACHE_SSL_CERT': '%s/%s.crt' % (dir['base'], name),
'APACHE_SSL_KEY': '%s/%s.key' % (dir['base'], name),
'APACHE_LOGGING_PATH': '/dev/null',
}
else:
environment = {
'APACHE_WEBROOT': pathOfServerUnderTest,
'APACHE_LOGGING_PATH': '/dev/null',
}
return [{
'name': name,
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': environment,
'command': [
'/usr/local/bin/apachectl',
'-e',
'debug',
'-D',
'FOREGROUND'
]
}]
def getShortDbNameAndVersion(db):
return '%s%s' % (getDbType(db), getDbVersion(db))
def getDbName(db):
return db.partition(':')[0]
def getDbVersion(db):
return db.partition(':')[2]
def getDbUsername(db):
name = getDbName(db)
# The Oracle image has the Db Username hardcoded
if name == 'oracle':
return 'autotest'
return 'owncloud'
def getDbPassword(db):
name = getDbName(db)
# The Oracle image has the Db Password hardcoded
if name == 'oracle':
return 'owncloud'
return 'owncloud'
def getDbRootPassword():
return 'owncloud'
def getDbDatabase(db):
name = getDbName(db)
# The Oracle image has the Db Name hardcoded
if name == 'oracle':
return 'XE'
return 'owncloud'
def getDbType(db):
dbName = getDbName(db)
if dbName == 'postgres':
return 'pgsql'
if dbName == 'oracle':
return 'oci'
return dbName
def cacheRestore():
return [{
'name': 'cache-restore',
'image': 'plugins/s3-cache:1',
'pull': 'always',
'settings': {
'access_key': {
'from_secret': 'cache_s3_access_key'
},
'endpoint': {
'from_secret': 'cache_s3_endpoint'
},
'restore': True,
'secret_key': {
'from_secret': 'cache_s3_secret_key'
}
},
'when': {
'instance': [
'drone.owncloud.services',
'drone.owncloud.com'
],
}
}]
def cacheClearOnEventPush(phpVersion):
return [{
'name': 'cache-clear',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'rm -Rf %s/.cache/composer' % dir['server'],
'rm -Rf %s/.cache/yarn' % dir['server'],
],
'when': {
'event': [
'push',
],
'instance': [
'drone.owncloud.services',
'drone.owncloud.com'
],
}
}]
def cacheRebuildOnEventPush():
return [{
'name': 'cache-rebuild',
'image': 'plugins/s3-cache:1',
'pull': 'always',
'settings': {
'access_key': {
'from_secret': 'cache_s3_access_key'
},
'endpoint': {
'from_secret': 'cache_s3_endpoint'
},
'mount': [
'.cache'
],
'rebuild': True,
'secret_key': {
'from_secret': 'cache_s3_secret_key'
}
},
'when': {
'event': [
'push',
],
'instance': [
'drone.owncloud.services',
'drone.owncloud.com'
],
}
}]
def cacheFlushOnEventPush():
return [{
'name': 'cache-flush',
'image': 'plugins/s3-cache:1',
'pull': 'always',
'settings': {
'access_key': {
'from_secret': 'cache_s3_access_key'
},
'endpoint': {
'from_secret': 'cache_s3_endpoint'
},
'flush': True,
'flush_age': '14',
'secret_key': {
'from_secret': 'cache_s3_secret_key'
}
},
'when': {
'event': [
'push',
],
'instance': [
'drone.owncloud.services',
'drone.owncloud.com'
],
}
}]
def composerInstall(phpVersion):
return [{
'name': 'composer-install',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': {
'COMPOSER_HOME': '%s/.cache/composer' % dir['server']
},
'commands': [
'make install-composer-deps'
]
}]
def vendorbinCodestyle(phpVersion):
return [{
'name': 'vendorbin-codestyle',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': {
'COMPOSER_HOME': '%s/.cache/composer' % dir['server']
},
'commands': [
'make vendor-bin-codestyle'
]
}]
def vendorbinCodesniffer(phpVersion):
return [{
'name': 'vendorbin-codesniffer',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': {
'COMPOSER_HOME': '%s/.cache/composer' % dir['server']
},
'commands': [
'make vendor-bin-codesniffer'
]
}]
def vendorbinPhan(phpVersion):
return [{
'name': 'vendorbin-phan',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': {
'COMPOSER_HOME': '%s/.cache/composer' % dir['server']
},
'commands': [
'make vendor-bin-phan'
]
}]
def vendorbinPhpstan(phpVersion):
return [{
'name': 'vendorbin-phpstan',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': {
'COMPOSER_HOME': '%s/.cache/composer' % dir['server']
},
'commands': [
'make vendor-bin-phpstan'
]
}]
def vendorbinBehat():
return [{
'name': 'vendorbin-behat',
'image': 'owncloudci/php:7.4',
'pull': 'always',
'environment': {
'COMPOSER_HOME': '%s/.cache/composer' % dir['server']
},
'commands': [
'make vendor-bin-behat'
]
}]
def yarnInstall(phpVersion):
return [{
'name': 'yarn-install',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': {
'NPM_CONFIG_CACHE': '%s/.cache/npm' % dir['server'],
'YARN_CACHE_FOLDER': '%s/.cache/yarn' % dir['server'],
'bower_storage__packages': '%s/.cache/bower' % dir['server'],
},
'commands': [
'make install-nodejs-deps'
]
}]
def davInstall(phpVersion, scriptPath):
return [{
'name': 'dav-install',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'bash %s/install.sh' % scriptPath
]
}]
def setupLocalStorage(phpVersion):
return [{
'name': 'setup-storage',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': {
'OC_PASS': '123456',
},
'commands': [
'mkdir -p %s/work/local_storage' % dir['server'],
'php occ app:enable files_external',
'php occ config:system:set files_external_allow_create_new_local --value=true',
'php occ config:app:set core enable_external_storage --value=yes',
'php occ files_external:create local_storage local null::null -c datadir=%s/work/local_storage' % dir['server'],
'php occ user:add --password-from-env user1',
]
}]
def createShare(phpVersion):
return [{
'name': 'create-share',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'curl -k -s -u user1:123456 -X MKCOL "https://server/remote.php/webdav/new_folder"',
'curl -k -s -u user1:123456 "https://server/ocs/v2.php/apps/files_sharing/api/v1/shares" --data "path=/new_folder&shareType=0&permissions=15&name=new_folder&shareWith=admin"',
'echo -n "PUBLIC_TOKEN=" > .env',
'curl -k -s -u user1:123456 "https://server/ocs/v2.php/apps/files_sharing/api/v1/shares" --data "path=/new_folder&shareType=3&permissions=15&name=new_folder" | grep token | cut -d">" -f2 | cut -d"<" -f1 >> .env',
]
}]
def installExtraApps(phpVersion, extraApps, pathOfServerUnderTest):
commandArray = []
for app, command in extraApps.items():
commandArray.append('ls %s/apps/%s || git clone https://github.com/owncloud/%s.git %s/apps/%s' % (pathOfServerUnderTest, app, app, pathOfServerUnderTest, app))
if (command != ''):
commandArray.append('cd %s/apps/%s' % (pathOfServerUnderTest, app))
commandArray.append(command)
commandArray.append('cd %s' % pathOfServerUnderTest)
commandArray.append('php occ a:l')
commandArray.append('php occ a:e %s' % app)
commandArray.append('php occ a:l')
if (commandArray == []):
return []
return [{
'name': 'install-extra-apps',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': commandArray
}]
def databaseServiceForFederation(db, suffix):
dbName = getDbName(db)
if dbName not in ['mariadb', 'mysql']:
print('Not implemented federated database for ', dbName)
return []
service = {
'name': dbName + suffix,
'image': db,
'pull': 'always',
'environment': {
'MYSQL_USER': getDbUsername(db),
'MYSQL_PASSWORD': getDbPassword(db),
'MYSQL_DATABASE': getDbDatabase(db) + suffix,
'MYSQL_ROOT_PASSWORD': getDbRootPassword()
}
}
if (db == 'mysql:8.0'):
service['command'] = ['--default-authentication-plugin=mysql_native_password']
return [service]
def installServer(phpVersion, db, logLevel = '2', ssl = False, federatedServerNeeded = False, proxyNeeded = False):
return [{
'name': 'install-server',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'environment': {
'DB_TYPE': getDbName(db),
'DB_USERNAME': getDbUsername(db),
'DB_PASSWORD': getDbPassword(db),
'DB_NAME': getDbDatabase(db)
},
'commands': [
'bash tests/drone/install-server.sh',
'php occ a:l',
'php occ config:system:set trusted_domains 1 --value=server',
] + ([
'php occ config:system:set trusted_domains 2 --value=federated',
'php occ config:system:set csrf.disabled --value=true'
] if federatedServerNeeded else []) + [
] + ([
'php occ config:system:set trusted_domains 3 --value=proxy'
] if proxyNeeded else []) + [
'php occ log:manage --level %s' % logLevel,
'php occ config:list',
] + ([
'php occ security:certificates:import %s/server.crt' % dir['base'],
] if ssl else []) + ([
'php occ security:certificates:import %s/federated.crt' % dir['base'],
] if federatedServerNeeded and ssl else []) + [
'php occ security:certificates',
]
}]
def installAndConfigureFederated(ctx, federatedServerVersion, phpVersion, logLevel, protocol, db, dbSuffix = 'fed'):
return [
installFederated(ctx, federatedServerVersion, db, dbSuffix),
configureFederated(phpVersion, logLevel, protocol)
]
def installFederated(ctx, federatedServerVersion, db, dbSuffix = 'fed'):
host = getDbName(db)
dbType = host
username = getDbUsername(db)
password = getDbPassword(db)
database = getDbDatabase(db) + dbSuffix
if host == 'mariadb':
dbType = 'mysql'
elif host == 'postgres':
dbType = 'pgsql'
elif host == 'oracle':
dbType = 'oci'
installerSettings = {
'core_path': dir["federated"],
'db_type': dbType,
'db_name': database,
'db_host': host + dbSuffix,
'db_username': username,
'db_password': password
}
if (federatedServerVersion == 'git'):
if (ctx.build.source_repo == ctx.repo.slug):
# The PR comes from a branch that is in the same repo
# So use that branch for installing the federated server
installerSettings['git_reference'] = ctx.build.source
else:
# The PR comes from a branch that is in some other repo
# e.g. a community contribution from a fork
# Rather than jumping through hoops to find that code,
# install the federated server from owncloud/core master
installerSettings['git_reference'] = 'master'
else:
installerSettings['version'] = federatedServerVersion
return {
'name': 'install-federated',
'image': 'owncloudci/core',
'pull': 'always',
'settings': installerSettings
}
def configureFederated(phpVersion, logLevel, protocol):
return {
'name': 'configure-federated',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'cd %s' % dir["federated"],
'php occ a:l',
'php occ a:e testing',
'php occ a:l',
'php occ config:system:set trusted_domains 1 --value=server',
'php occ config:system:set trusted_domains 2 --value=federated',
'php occ log:manage --level %s' % logLevel,
'php occ config:list',
'echo "export TEST_SERVER_FED_URL=%s://federated" > %s/saved-settings.sh' % (protocol, dir['base']),
'php occ security:certificates:import %s/server.crt' % dir['base'],
'php occ security:certificates:import %s/federated.crt' % dir['base'],
'php occ security:certificates',
]
}
def setupCeph(phpVersion, cephS3):
if not cephS3:
return []
return [{
'name': 'setup-ceph',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'wait-for-it -t 600 ceph:80',
'cd %s/apps/files_primary_s3' % dir['server'],
'cp tests/drone/ceph.config.php %s/config' % dir['server'],
'cd /var/www/owncloud/server',
'./apps/files_primary_s3/tests/drone/create-bucket.sh',
]
}]
def setupScality(phpVersion, scalityS3):
if type(scalityS3) == "bool":
if scalityS3:
# specify an empty dict that will get the defaults
scalityS3 = {}
else:
return []
specialConfig = '.' + scalityS3['config'] if 'config' in scalityS3 else ''
configFile = 'scality%s.config.php' % specialConfig
createExtraBuckets = scalityS3['createExtraBuckets'] if 'createExtraBuckets' in scalityS3 else False
return [{
'name': 'setup-scality',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'wait-for-it -t 600 scality:8000',
'cp %s/apps/files_primary_s3/tests/drone/%s %s/config' % (dir['server'], configFile, dir['server']),
'php occ s3:create-bucket owncloud --accept-warning'
] + ([
'for I in $(seq 1 9); do php ./occ s3:create-bucket owncloud$I --accept-warning; done',
] if createExtraBuckets else [])
}]
def fixPermissions(phpVersion, federatedServerNeeded, pathOfServerUnderTest = dir['server']):
return [{
'name': 'fix-permissions',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'chown -R www-data %s' % pathOfServerUnderTest
] + ([
'chown -R www-data %s' % dir["federated"]
] if federatedServerNeeded else [])
}]
def waitForServer(phpVersion, federatedServerNeeded):
return [{
'name': 'wait-for-server',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'wait-for-it -t 600 server:80'
] + ([
'wait-for-it -t 600 federated:80'
] if federatedServerNeeded else [])
}]
def owncloudLog(server, folder):
return [{
'name': 'owncloud-log-%s' % server,
'image': 'owncloud/ubuntu:18.04',
'pull': 'always',
'detach': True,
'commands': [
'tail -f %s/data/owncloud.log' % folder
]
}]
def owncloudDockerService(ocDockerService):
if not ocDockerService:
return []
return [{
'name': 'oc-server',
'image': 'owncloud/server',
'pull': 'always',
'environment': {
'OWNCLOUD_VERSION': '10.7',
'OWNCLOUD_DOMAIN': 'oc-server',
'OWNCLOUD_ADMIN_USERNAME': 'admin',
'OWNCLOUD_ADMIN_PASSWORD': 'admin',
'HTTP_PORT': '8080',
'OWNCLOUD_REDIS_ENABLED': 'true',
'OWNCLOUD_REDIS_HOST': 'redis'
},
}]
def redisService(redisService):
if not redisService:
return []
return [{
'name': 'redis',
'image': 'webhippie/redis:latest',
'pull': 'always',
'environment': {
'REDIS_DATABASES': 1
}
}]
def dependsOn(earlierStages, nextStages):
for earlierStage in earlierStages:
for nextStage in nextStages:
nextStage['depends_on'].append(earlierStage['name'])
def installCoreFromTarball(version, db, logLevel = '2', ssl = False, federatedServerNeeded = False, proxyNeeded = False, pathOfServerUnderTest = '/drone/core'):
host = getDbName(db)
dbType = host
username = getDbUsername(db)
password = getDbPassword(db)
database = getDbDatabase(db)
if host == 'mariadb':
dbType = 'mysql'
if host == 'postgres':
dbType = 'pgsql'
if host == 'oracle':
dbType = 'oci'
return [{
'name': 'install-tarball',
'image': 'owncloudci/core',
'pull': 'always',
'settings': {
'version': version,
'core_path': pathOfServerUnderTest,
'db_type': dbType,
'db_name': database,
'db_host': host,
'db_username': username,
'db_password': password
},
},{
'name': 'configure-tarball',
'image': 'owncloudci/php:7.4',
'pull': 'always',
'commands': [
'cd %s' % pathOfServerUnderTest,
'php occ a:l',
'php occ a:e testing',
'php occ a:l',
'php occ config:system:set trusted_domains 1 --value=server',
] + ([
'php occ config:system:set trusted_domains 2 --value=federated'
] if federatedServerNeeded else []) + [
] + ([
'php occ config:system:set trusted_domains 3 --value=proxy'
] if proxyNeeded else []) + [
'php occ log:manage --level %s' % logLevel,
'php occ config:list',
] + ([
'php occ security:certificates:import %s/server.crt' % dir['base'],
] if ssl else []) + ([
'php occ security:certificates:import %s/federated.crt' % dir['base'],
] if federatedServerNeeded and ssl else []) + [
'php occ security:certificates',
]
}]
def installFederatedFromTarball(federatedServerVersion, phpVersion, logLevel, protocol, db, dbSuffix = '-federated'):
host = getDbName(db)
dbType = host
username = getDbUsername(db)
password = getDbPassword(db)
database = getDbDatabase(db) + dbSuffix
if host == 'mariadb':
dbType = 'mysql'
elif host == 'postgres':
dbType = 'pgsql'
elif host == 'oracle':
dbType = 'oci'
return [
{
'name': 'install-federated',
'image': 'owncloudci/core',
'pull': 'always',
'settings': {
'version': federatedServerVersion,
'core_path': dir["federated"],
'db_type': 'mysql',
'db_name': database,
'db_host': host + dbSuffix,
'db_username': username,
'db_password': password
},
},
{
'name': 'configure-federation',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'echo "export TEST_SERVER_FED_URL=%s://federated" > %s/saved-settings.sh' % (protocol, dir['base']),
'cd %s' % dir["federated"],
'php occ a:l',
'php occ a:e testing',
'php occ a:l',
'php occ config:system:set trusted_domains 1 --value=server',
'php occ config:system:set trusted_domains 2 --value=federated',
'php occ log:manage --level %s' % logLevel,
'php occ config:list',
'php occ security:certificates:import %s/server.crt' % dir['base'],
'php occ security:certificates:import %s/federated.crt' % dir['base'],
'php occ security:certificates',
]
}
]
def installTestRunner(ctx, phpVersion):
return [{
'name': 'install-testrunner',
'image': 'owncloudci/php:%s' % phpVersion,
'pull': 'always',
'commands': [
'mkdir /tmp/testrunner',
'git clone -b %s --depth=1 https://github.com/owncloud/core.git /tmp/testrunner' % ctx.build.source if ctx.build.event == 'pull_request' else 'master',
'rsync -aIX /tmp/testrunner/tests %s/tests' % dir['server'],
]
}]