Implement efficient destroy of job_trace_chunks

app/models/ci/job_trace_chunk.rb

@@ -19,6 +19,30 @@ module Ci
       db: 2
     }
 
+    def self.delayed_cleanup_blk
+      ids = all.redis.pluck(:job_id, :chunk_index).map do |data|
+        "gitlab:ci:trace:#{data.first}:chunks:#{data.second}:data"
+      end
+
+      puts "before cleanup: #{ids.count}"
+
+      Proc.new do
+        puts "after cleanup: #{ids.count}"
+        Gitlab::Redis::SharedState.with do |redis|
+          redis.del(ids)
+        end unless ids.empty?
+
+        true
+      end
+    end
+
+    def self.fast_destroy_all
+      delayed_cleanup_blk.tap do |cleanup|
+        delete_all
+        cleanup.call
+      end
+    end
+
     def data
       if redis?
         redis_data

app/models/project.rb

@@ -209,12 +209,21 @@ class Project < ActiveRecord::Base
   has_many :commit_statuses
   has_many :pipelines, class_name: 'Ci::Pipeline', inverse_of: :project
 
+  # This has to be defined before `has_many :builds, depenedent: :destroy`,
+  # otherwise we will not delete any data, due to trace chunks
+  # going through :builds
+  before_destroy do
+    puts "destroying all chunks"
+    self.run_after_commit(&build_trace_chunks.delayed_cleanup_blk)
+  end
+
   # Ci::Build objects store data on the file system such as artifact files and
   # build traces. Currently there's no efficient way of removing this data in
   # bulk that doesn't involve loading the rows into memory. As a result we're
   # still using `dependent: :destroy` here.
   has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName'
+  has_many :build_trace_chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id, through: :builds, source: :chunks
   has_many :runner_projects, class_name: 'Ci::RunnerProject'
   has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
   has_many :variables, class_name: 'Ci::Variable'

app/workers/build_finished_worker.rb

@@ -7,6 +7,7 @@ class BuildFinishedWorker
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
       # Swap all trace chunks to Database from Redis
+      # TODO: Do we need that?
       build.chunks.redis.map(&:use_database!)
 
       # We execute that in sync as this access the files in order to access local data, and reduce IO

lib/gitlab/ci/trace.rb

@@ -100,7 +100,7 @@ module Gitlab
           FileUtils.rm(trace_path, force: true)
         end
 
-        job.chunks.delete_all
+        job.chunks.fast_destroy_all
         job.erase_old_trace!
       end
 

lib/gitlab/ci/trace/chunked_io.rb

@@ -140,7 +140,7 @@ module Gitlab
           @size = offset
 
           # remove all next chunks
-          job_chunks.where('chunk_index > ?', chunk_index).delete_all
+          job_chunks.where('chunk_index > ?', chunk_index).fast_destroy_all
 
           # truncate current chunk
           current_chunk.truncate(chunk_offset) if chunk_offset != 0
@@ -156,8 +156,8 @@ module Gitlab
           true
         end
 
-        def delete!
-          job_chunks.delete_all
+        def destroy!
+          job_chunks.fast_destroy_all
           @tell = @size = 0
         ensure
           invalidate_chunk_cache