https://github.com/mperham/sidekiq
Tip revision: c2e819769a9d765b93c66f8ae0e2737885303ae1 authored by Mike Perham on 08 January 2019, 17:23:09 UTC
bump, changes
bump, changes
Tip revision: c2e8197
sidekiqload
#!/usr/bin/env ruby
# Quiet some warnings we see when running in warning mode:
# RUBYOPT=-w bundle exec sidekiq
$TESTING = false
#require 'ruby-prof'
Bundler.require(:default)
require_relative '../lib/sidekiq/cli'
require_relative '../lib/sidekiq/launcher'
include Sidekiq::Util
Sidekiq.configure_server do |config|
#config.options[:concurrency] = 1
config.redis = { db: 13 }
config.options[:queues] << 'default'
config.logger.level = Logger::ERROR
config.average_scheduled_poll_interval = 2
config.reliable! if defined?(Sidekiq::Pro)
end
class LoadWorker
include Sidekiq::Worker
sidekiq_options retry: 1
sidekiq_retry_in do |x|
1
end
def perform(idx)
#raise idx.to_s if idx % 100 == 1
end
end
# brew tap shopify/shopify
# brew install toxiproxy
# gem install toxiproxy
#require 'toxiproxy'
# simulate a non-localhost network for realer-world conditions.
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
#Toxiproxy.populate([{
#"name": "redis",
#"listen": "127.0.0.1:6380",
#"upstream": "127.0.0.1:6379"
#}])
self_read, self_write = IO.pipe
%w(INT TERM TSTP TTIN).each do |sig|
begin
trap sig do
self_write.puts(sig)
end
rescue ArgumentError
puts "Signal #{sig} not supported"
end
end
Sidekiq.redis {|c| c.flushdb}
def handle_signal(launcher, sig)
Sidekiq.logger.debug "Got #{sig} signal"
case sig
when 'INT'
# Handle Ctrl-C in JRuby like MRI
# http://jira.codehaus.org/browse/JRUBY-4637
raise Interrupt
when 'TERM'
# Heroku sends TERM and then waits 10 seconds for process to exit.
raise Interrupt
when 'TSTP'
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
launcher.quiet
when 'TTIN'
Thread.list.each do |thread|
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
if thread.backtrace
Sidekiq.logger.warn thread.backtrace.join("\n")
else
Sidekiq.logger.warn "<no backtrace available>"
end
end
end
end
def Process.rss
`ps -o rss= -p #{Process.pid}`.chomp.to_i
end
iter = 10
count = 10_000
iter.times do
arr = Array.new(count) do
[]
end
count.times do |idx|
arr[idx][0] = idx
end
Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
end
Sidekiq.logger.error "Created #{count*iter} jobs"
Monitoring = Thread.new do
watchdog("monitor thread") do
while true
sleep 1
qsize, retries = Sidekiq.redis do |conn|
conn.pipelined do
conn.llen "queue:default"
conn.zcard "retry"
end
end.map(&:to_i)
total = qsize + retries
#GC.start
Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
if total == 0
Sidekiq.logger.error("Done")
exit(0)
end
end
end
end
begin
#RubyProf::exclude_threads = [ Monitoring ]
#RubyProf.start
fire_event(:startup)
#Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
#Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
launcher = Sidekiq::Launcher.new(Sidekiq.options)
launcher.run
while readable_io = IO.select([self_read])
signal = readable_io.first[0].gets.strip
handle_signal(launcher, signal)
end
#end
rescue SystemExit => e
#Sidekiq.logger.error("Profiling...")
#result = RubyProf.stop
#printer = RubyProf::GraphHtmlPrinter.new(result)
#printer.print(File.new("output.html", "w"), :min_percent => 1)
# normal
rescue => e
raise e if $DEBUG
STDERR.puts e.message
STDERR.puts e.backtrace.join("\n")
exit 1
end