Revision 34252c922b761a8f4d1debc21d784a835a58ed24 authored by Antoine Musso on 02 June 2017, 20:05:18 UTC, committed by Antoine Musso on 02 June 2017, 20:05:18 UTC
Make ::kibana class to accept a 'settings' hash which is then used to generate the kibana.yaml configuration file. Kept logging.quiet to default to true by using Puppet stdlib merge() https://www.elastic.co/guide/en/kibana/current/settings.html Change-Id: I4083e7315e26b71efb3c95fa5c5cab5a47e30777
1 parent 76440f6
common.yaml
# General variables that once would have been in realm.pp
datacenters:
- eqiad
- codfw
- esams
- ulsfo
# Main statsd instance
statsd: statsd.eqiad.wmnet:8125
# NOTE: Do *not* add new clusters *per site* anymore,
# the site name will automatically be appended now,
# and a different IP prefix will be used.
ganglia_clusters:
decommissioned:
description: "Decommissioned servers"
id: 1
sites: {}
lvs:
description: "LVS loadbalancers"
id: 2
sites:
eqiad: []
codfw: []
esams: []
ulsfo: []
puppet:
description: "Puppetmasters"
id: 3
sites:
eqiad: []
codfw: []
search:
description: "Search"
id: 4
sites: {}
mysql:
description: "MySQL"
id: 5
sites:
eqiad: []
codfw: []
etcd:
description: "Etcd"
id: 6
sites:
eqiad: []
codfw: []
eventbus:
description: "Eventbus"
id: 7
sites:
eqiad: []
misc:
description: "Miscellaneous"
id: 8
sites:
eqiad: []
codfw: []
esams: []
kubernetes:
description: "Kubernetes"
id: 9
sites:
eqiad: []
codfw: []
appserver:
description: "Application servers"
id: 11
sites:
eqiad: []
codfw: []
imagescaler:
description: "Image scalers"
id: 12
sites:
eqiad: []
codfw: []
api_appserver:
description: "API application servers"
id: 13
sites:
eqiad: []
codfw: []
pdf:
description: "PDF servers"
id: 15
sites:
eqiad: []
cache_text:
description: "Text caches"
id: 20
sites:
eqiad: []
codfw: []
esams: []
ulsfo: []
cache_upload:
description: "Upload caches"
id: 22
sites:
eqiad: []
codfw: []
esams: []
ulsfo: []
payments:
description: "Fundraiser payments"
id: 23
sites: {}
ssl:
description: "SSL cluster"
id: 26
sites: {}
swift:
description: "Swift"
id: 27
sites:
eqiad: []
codfw: []
esams: []
labvirt:
description: "Labs virt hosts"
id: 29
sites:
eqiad: []
labs:
description: "Labs services"
id: 30
sites:
eqiad: []
jobrunner:
description: "Jobrunners"
id: 31
sites:
eqiad: []
codfw: []
analytics:
description: "Analytics cluster"
id: 32
sites:
eqiad: []
memcached:
description: "Memcached"
id: 33
sites:
eqiad: []
codfw: []
videoscaler:
description: "Video scalers"
id: 34
sites:
eqiad: []
codfw: []
fundraising:
description: "Fundraising"
id: 35
sites:
eqiad:
- pay-lvs1001.frack.eqiad.wmnet
- pay-lvs1002.frack.eqiad.wmnet
ceph: # Not used anymore
description: "Ceph"
id: 36
sites: {}
parsoid:
description: "Parsoid"
id: 37
sites:
eqiad: []
codfw: []
redis:
description: "Redis"
id: 39
sites:
eqiad: []
codfw: []
labsnfs:
description: "Labs NFS cluster"
id: 40
sites:
eqiad: []
codfw: []
cache_misc:
description: "Misc Web caching cluster"
id: 41
sites:
eqiad: []
codfw: []
esams: []
ulsfo: []
elasticsearch:
description: "Elasticsearch cluster"
id: 42
sites:
eqiad: []
codfw: []
logstash:
description: "Logstash cluster"
id: 43
sites:
eqiad: []
rcstream:
description: "RCStream cluster"
id: 44
sites:
eqiad: []
analytics_kafka:
description: "Analytics Kafka cluster"
id: 45
sites:
eqiad: []
sca:
description: "Service Cluster A"
id: 46
sites:
eqiad: []
codfw: []
restbase:
description: "Restbase"
id: 48
sites:
eqiad: []
codfw: []
wdqs:
description: "Wikidata Query Service"
id: 49
sites:
eqiad: []
maps:
description: "Maps Cluster"
id: 50
sites:
codfw: []
cache_maps:
description: "Maps caches"
id: 51
sites:
eqiad: []
codfw: []
esams: []
ulsfo: []
ganeti:
description: "Ganeti Virt cluster"
id: 52
sites:
eqiad: []
codfw: []
scb:
description: "Service Cluster B"
id: 53
sites:
eqiad: []
codfw: []
aqs:
description: "Analytics Query Service"
id: 54
sites:
eqiad: []
restbase_test:
description: "Restbase test"
id: 55
sites:
eqiad: []
codfw: []
relforge:
description: "Elasticsearch relforge cluster"
id: 56
sites:
eqiad: []
labtestvirt:
description: "Labtest virt hosts"
id: 57
sites:
codfw: []
labtest:
description: "Labtest services"
id: 58
sites:
codfw: []
restbase_dev:
description: "Services development test"
id: 59
sites:
eqiad: []
puppetmaster: "puppet"
active_labstore_host: 'labstore1001'
statistics_servers:
- stat1002.eqiad.wmnet
- stat1003.eqiad.wmnet
- dataset1001.wikimedia.org
- thorium.eqiad.wmnet
dataset_clients_snapshots:
- snapshot1001.eqiad.wmnet
- snapshot1005.eqiad.wmnet
- snapshot1006.eqiad.wmnet
- snapshot1007.eqiad.wmnet
dataset_clients_other:
- stat1002.eqiad.wmnet
- stat1003.eqiad.wmnet
# Schemas names that match this regex
# will not be produced to the eventlogging-valid-mixed
# topic. This allows us to keep high volume schema
# events out of the MySQL consumer.
# This is in common.yaml so that this will be used
# for eventlogging in labs as well as production.
# Currently this blacklists the following schemas:
# * Analytics so we can use it for testing throughput
# * CentralNoticeBannerHistory because it can't be inserted into mysql with its array property
# * ImageMetrics see T141407, these tables are no longer used and yet some cached code is still sending events
eventlogging_valid_mixed_schema_blacklist: ^(Analytics|CentralNoticeBannerHistory|ImageMetricsLoadingTime|ImageMetricsCorsSupport)$
# Used to specify writer handler for eventlogging processor producer.
eventlogging_kafka_producer_scheme: "kafka-confluent://"
# LABS
labs_tld: "wmflabs"
labs_private_ips_reverse_dns: "68.10.in-addr.arpa"
labs_designate_hostname: &labsdesignatehostname "labservices1001.wikimedia.org"
labs_designate_hostname_secondary: &labs_designate_hostname_secondary "labservices1002.wikimedia.org"
labs_nova_api_host: &labsnovaapihost "labnet1001.eqiad.wmnet"
labs_nova_network_host: &labsnovanetworkhost "labnet1001"
labs_nova_network_ip: &labsnovanetworkip "10.64.20.25"
status_wiki_host_master: 'wikitech.wikimedia.org'
# By default, don't allow projects to allocate public IPs; this way we can
# let users have network admin rights, for firewall rules and such, and can
# give them public ips by increasing their quota
novaconfig:
network_host: *labsnovanetworkip
api_host: *labsnovaapihost
db_name: 'nova'
api_db_name: 'novaapi'
db_user: 'nova'
ceilometer_db_name: 'ceilometer'
ldap_base_dn: 'dc=wikimedia,dc=org'
ldap_user_dn: 'uid=novaadmin,ou=people,dc=wikimedia,dc=org'
ldap_proxyagent: 'cn=proxyagent,ou=profile,dc=wikimedia,dc=org'
puppet_db_name: 'puppet'
puppet_db_user: 'puppet'
quota_floating_ips: '0'
libvirt_type: 'kvm'
my_ip: "%{::ipaddress}"
network_public_interface: 'eth0'
network_flat_interface: 'eth1.1102'
network_flat_tagged_base_interface: 'eth1'
network_flat_interface_vlan: '1102'
flat_network_bridge: 'br1102'
fixed_range: '10.68.16.0/21'
dhcp_start: '10.68.16.4'
network_public_ip: '208.80.155.255'
dmz_cidr: '208.80.155.0/22,10.0.0.0/8'
keystoneconfig:
db_name: 'keystone'
db_user: 'keystone'
ldap_base_dn: 'dc=wikimedia,dc=org'
ldap_user_dn: 'uid=novaadmin,ou=people,dc=wikimedia,dc=org'
ldap_user_id_attribute: 'uid'
ldap_tenant_id_attribute: 'cn'
ldap_user_name_attribute: 'cn'
ldap_tenant_name_attribute: 'cn'
ldap_proxyagent : 'cn=proxyagent,ou=profile,dc=wikimedia,dc=org'
auth_protocol: 'http'
auth_port: '35357'
public_port: '5000'
db_host: 'm5-master.eqiad.wmnet'
ldap_host: 'ldap-labs.eqiad.wikimedia.org'
token_driver: 'normal'
glanceconfig:
db_host: 'm5-master.eqiad.wmnet'
bind_ip: "%{::ipaddress}"
# IPs assigned here MUST be reserved beforehand so they don't collide with labs VMs.
# To reserve, on the labs controller run $ sudo nova-manage fixed reserve <ip>
# (If using Neutron, all bets are off and this comment is wrong.)
# Note also that nova-network is not great about refreshing dnsmasq,
# so changes to this table may require a manual kill of dnsmasq and restart
# of nova-network.
labs_metal:
promethium: {MAC: '90:b1:1c:2d:6f:0c', IPv4: 10.68.16.2, project: wikitextexp }
# List of all zookeeper clusters in production.
zookeeper_clusters:
main-eqiad:
hosts:
conf1001.eqiad.wmnet: '1101'
conf1002.eqiad.wmnet: '1102'
conf1003.eqiad.wmnet: '1103'
main-codfw:
hosts:
conf2001.codfw.wmnet: '2001'
conf2002.codfw.wmnet: '2002'
conf2003.codfw.wmnet: '2003'
# ZK cluster for Druid (in Analytics cluster),
# colocated with Druid.
druid-eqiad:
hosts:
druid1001.eqiad.wmnet: '1001'
druid1002.eqiad.wmnet: '1002'
druid1003.eqiad.wmnet: '1003'
kafka_clusters:
# This is the analytics Kafka cluster, named just 'eqiad' for
# historical reasons.
eqiad:
# Optional api_version indicates the Kafka API version the
# brokers are running. Clients can use this to override
# version discovery for versions of Kafka where the version
# request API doesn't exist (< 0.10). Once all brokers
# are on 0.10, this shouldn't be needed.
api_version: 0.9
zookeeper_cluster_name: main-eqiad
brokers:
kafka1012.eqiad.wmnet:
id: 12 # Row A
kafka1013.eqiad.wmnet:
id: 13 # Row A
kafka1014.eqiad.wmnet:
id: 14 # Row C
kafka1018.eqiad.wmnet:
id: 18 # Row D
kafka1020.eqiad.wmnet:
id: 20 # Row D
kafka1022.eqiad.wmnet:
id: 22 # Row C
main-eqiad:
api_version: 0.9
zookeeper_cluster_name: main-eqiad
brokers:
kafka1001.eqiad.wmnet:
id: 1001
kafka1002.eqiad.wmnet:
id: 1002
kafka1003.eqiad.wmnet:
id: 1003
main-codfw:
api_version: 0.9
zookeeper_cluster_name: main-codfw
brokers:
kafka2001.codfw.wmnet:
id: 2001
kafka2002.codfw.wmnet:
id: 2002
kafka2003.codfw.wmnet:
id: 2003
deployment_server: tin.eqiad.wmnet
install_server: install1002.wikimedia.org
install_server_failover: install2002.wikimedia.org
# Etcd client global configuration
etcd_client_srv_domain: "conftool.%{::site}.wmnet"
etcd_host: ''
etcd_port: ''
# Conftool global prefix (will be per-dc)
conftool_prefix: "/conftool/v1"
# Logging: logstash, udp2log
# TODO: setup something to loadbalance the logstash hosts rather than relying
# on a single node in the cluster.
logstash_syslog: "logstash1001.eqiad.wmnet:10514"
udp2log_aggregator: "udplog:8420"
tcpircbot_host: 'icinga.wikimedia.org'
tcpircbot_port: 9200
Computing file changes ...
