157 lines
4.3 KiB
Ruby
157 lines
4.3 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module Gitlab
|
|
module SidekiqLogging
|
|
class StructuredLogger
|
|
START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze
|
|
DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze
|
|
MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes
|
|
|
|
def call(job, queue)
|
|
started_time = get_time
|
|
base_payload = parse_job(job)
|
|
|
|
Sidekiq.logger.info log_job_start(base_payload)
|
|
|
|
yield
|
|
|
|
Sidekiq.logger.info log_job_done(job, started_time, base_payload)
|
|
rescue => job_exception
|
|
Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception)
|
|
|
|
raise
|
|
end
|
|
|
|
private
|
|
|
|
def base_message(payload)
|
|
"#{payload['class']} JID-#{payload['jid']}"
|
|
end
|
|
|
|
def add_instrumentation_keys!(job, output_payload)
|
|
output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS))
|
|
end
|
|
|
|
def log_job_start(payload)
|
|
payload['message'] = "#{base_message(payload)}: start"
|
|
payload['job_status'] = 'start'
|
|
|
|
# Old gitlab-shell messages don't provide enqueued_at/created_at attributes
|
|
enqueued_at = payload['enqueued_at'] || payload['created_at']
|
|
if enqueued_at
|
|
payload['scheduling_latency_s'] = elapsed_by_absolute_time(Time.iso8601(enqueued_at))
|
|
end
|
|
|
|
payload
|
|
end
|
|
|
|
def log_job_done(job, started_time, payload, job_exception = nil)
|
|
payload = payload.dup
|
|
add_instrumentation_keys!(job, payload)
|
|
|
|
elapsed_time = elapsed(started_time)
|
|
add_time_keys!(elapsed_time, payload)
|
|
|
|
message = base_message(payload)
|
|
|
|
if job_exception
|
|
payload['message'] = "#{message}: fail: #{payload['duration']} sec"
|
|
payload['job_status'] = 'fail'
|
|
payload['error_message'] = job_exception.message
|
|
payload['error'] = job_exception.class
|
|
payload['error_backtrace'] = backtrace_cleaner.clean(job_exception.backtrace)
|
|
else
|
|
payload['message'] = "#{message}: done: #{payload['duration']} sec"
|
|
payload['job_status'] = 'done'
|
|
end
|
|
|
|
convert_to_iso8601(payload, DONE_TIMESTAMP_FIELDS)
|
|
|
|
payload
|
|
end
|
|
|
|
def add_time_keys!(time, payload)
|
|
payload['duration'] = time[:duration].round(3)
|
|
payload['system_s'] = time[:stime].round(3)
|
|
payload['user_s'] = time[:utime].round(3)
|
|
payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0
|
|
payload['completed_at'] = Time.now.utc
|
|
end
|
|
|
|
def parse_job(job)
|
|
job = job.dup
|
|
|
|
# Add process id params
|
|
job['pid'] = ::Process.pid
|
|
|
|
job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS']
|
|
job['args'] = limited_job_args(job['args']) if job['args']
|
|
|
|
convert_to_iso8601(job, START_TIMESTAMP_FIELDS)
|
|
|
|
job
|
|
end
|
|
|
|
def convert_to_iso8601(payload, keys)
|
|
keys.each do |key|
|
|
payload[key] = format_time(payload[key]) if payload[key]
|
|
end
|
|
end
|
|
|
|
def elapsed_by_absolute_time(start)
|
|
(Time.now.utc - start).to_f.round(3)
|
|
end
|
|
|
|
def elapsed(t0)
|
|
t1 = get_time
|
|
{
|
|
duration: t1[:now] - t0[:now],
|
|
stime: t1[:times][:stime] - t0[:times][:stime],
|
|
utime: t1[:times][:utime] - t0[:times][:utime],
|
|
ctime: ctime(t1[:times]) - ctime(t0[:times])
|
|
}
|
|
end
|
|
|
|
def get_time
|
|
{
|
|
now: current_time,
|
|
times: Process.times
|
|
}
|
|
end
|
|
|
|
def ctime(times)
|
|
times[:cstime] + times[:cutime]
|
|
end
|
|
|
|
def current_time
|
|
Gitlab::Metrics::System.monotonic_time
|
|
end
|
|
|
|
def backtrace_cleaner
|
|
@backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new
|
|
end
|
|
|
|
def format_time(timestamp)
|
|
return timestamp if timestamp.is_a?(String)
|
|
|
|
Time.at(timestamp).utc.iso8601(3)
|
|
end
|
|
|
|
def limited_job_args(args)
|
|
return unless args.is_a?(Array)
|
|
|
|
total_length = 0
|
|
limited_args = args.take_while do |arg|
|
|
total_length += arg.to_json.length
|
|
|
|
total_length <= MAXIMUM_JOB_ARGUMENTS_LENGTH
|
|
end
|
|
|
|
limited_args.push('...') if total_length > MAXIMUM_JOB_ARGUMENTS_LENGTH
|
|
|
|
limited_args
|
|
end
|
|
end
|
|
end
|
|
end
|