From c4c8ca04052aaf7d37c2335066381b536df68427 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Wed, 20 Jan 2016 21:49:26 +0100
Subject: [PATCH] Added support for zip archives in pages

The ZIP archive size is calculated from artifacts metadata that should get uploaded for new artifacts
---
 app/services/projects/update_pages_service.rb |  41 ++++++++--
 spec/fixtures/pages.zip                       | Bin 0 -> 1851 bytes
 spec/fixtures/pages.zip.meta                  | Bin 0 -> 225 bytes
 spec/fixtures/pages_empty.zip                 | Bin 0 -> 160 bytes
 spec/fixtures/pages_empty.zip.meta            | Bin 0 -> 116 bytes
 .../projects/update_pages_worker_spec.rb      |  74 ++++++++++--------
 6 files changed, 78 insertions(+), 37 deletions(-)
 create mode 100644 spec/fixtures/pages.zip
 create mode 100644 spec/fixtures/pages.zip.meta
 create mode 100644 spec/fixtures/pages_empty.zip
 create mode 100644 spec/fixtures/pages_empty.zip.meta

diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb
index e1bb4c92e40..ceabd29fd52 100644
--- a/app/services/projects/update_pages_service.rb
+++ b/app/services/projects/update_pages_service.rb
@@ -2,6 +2,7 @@ module Projects
   class UpdatePagesService < BaseService
     BLOCK_SIZE = 32.kilobytes
     MAX_SIZE = 1.terabyte
+    SITE_PATH = 'public/'
 
     attr_reader :build
 
@@ -60,13 +61,42 @@ module Projects
     end
 
     def extract_archive!(temp_path)
+      if artifacts.ends_with?('.tar.gz') || artifacts.ends_with?('.tgz')
+        extract_tar_archive!(temp_path)
+      elsif artifacts.ends_with?('.zip')
+        extract_zip_archive!(temp_path)
+      else
+        raise 'unsupported artifacts format'
+      end
+    end
+
+    def extract_tar_archive!(temp_path)
       results = Open3.pipeline(%W(gunzip -c #{artifacts}),
                                %W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
-                               %W(tar -x -C #{temp_path} public/),
+                               %W(tar -x -C #{temp_path} #{SITE_PATH}),
                                err: '/dev/null')
       raise 'pages failed to extract' unless results.compact.all?(&:success?)
     end
 
+    def extract_zip_archive!(temp_path)
+      raise 'missing artifacts metadata' unless build.artifacts_metadata?
+
+      # Calculate page size after extract
+      public_entry = build.artifacts_metadata_entry(SITE_PATH, recursive: true)
+
+      if public_entry.total_size > max_size
+        raise "artifacts for pages are too large: #{total_size}"
+      end
+
+      # Requires UnZip at least 6.00 Info-ZIP.
+      # -n  never overwrite existing files
+      # We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories
+      site_path = File.join(SITE_PATH, '*')
+      unless system(*%W(unzip -n #{artifacts} #{site_path} -d #{temp_path}))
+        raise 'pages failed to extract'
+      end
+    end
+
     def deploy_page!(archive_public_path)
       # Do atomic move of pages
       # Move and removal may not be atomic, but they are significantly faster then extracting and removal
@@ -91,10 +121,11 @@ module Projects
 
     def blocks
       # Calculate dd parameters: we limit the size of pages
-      max_size = current_application_settings.max_pages_size.megabytes
-      max_size ||= MAX_SIZE
-      blocks = 1 + max_size / BLOCK_SIZE
-      blocks
+      1 + max_size / BLOCK_SIZE
+    end
+
+    def max_size
+      current_application_settings.max_pages_size.megabytes || MAX_SIZE
     end
 
     def tmp_path
diff --git a/spec/fixtures/pages.zip b/spec/fixtures/pages.zip
new file mode 100644
index 0000000000000000000000000000000000000000..9558fcd4b9428385e15ae386f473434d56d3da3f
GIT binary patch
literal 1851
zcmWIWW@h1H0D;mJeGf1LO0Wazg3_d%%w+ulpdt|<mxE!|w0?KyCeOnw7#SG2m>C!Z
z5sEVNQc^4QGD>oDe1p7K$r$q7c`jmg;DY!GtvxB?P6`wD9=mpi<4^UZtIz#+RvW!{
ze9X2}Cv@tobn}%<4lfRRw^Vwk-y^NhPAko)o20f0rpvzayghfxi#sp;e@vKmH8w-U
zFUcme?67;0_!ke&RmIABzvN=fe~Ic&yrwu!wBc0Zt0PmUYJ0A@K5wPVtcTLI?|p^d
zT17U@5k4_*>w3MV4E1wXO79As=$9+1w?y)VlH8n`4l+wteha$x{>7$?vo~JWYh9wb
z#C(<4DNV7oEbm#19UsJhTzOr?VA&V#mbI^g@1Oi5VJ4Sj)cf*6{pB`wzsB$P{ce^#
z_Hf_0QoO3&$1Ab#&=f1XC4O7xR47gVpIm0f%9HntCAIp?G^S)m7s+{#ub3sOTO=jF
z?pwIV^YPJdE;havb9Jiv4@v)>ak^OT`NB&M`+lFk8C!0s(Im$cr<5;L{&Z`}Ndq~b
z$y~oqM~Nm*ox$?-kENmX-)(%w{09r3I`SVmdPL-;R?iJt!Gh<^MV84;x9Xm~p8S`m
z_k0K6<yGEa+t=QmWEZ@7#hfn(@=Ev(8*hf|KKLef?9<n!f4=_PlJNTzKO$Ps0i)GR
zsP9oQ3ot$n1R1yy(OQ*Sl>hc1BQU-hniT%8&+~8qF-O5@2#kinXb6xQ0v{3rycwC~
zm~q!U5Q7C6-a3L<@OlWO33p8dQVJ@amNcp&G$HCGh@BAa*h)N*b`V(7_y=eT)J{<O
T7vRmx29jU~!nHuU0?Y#dcdi$E

literal 0
HcmV?d00001

diff --git a/spec/fixtures/pages.zip.meta b/spec/fixtures/pages.zip.meta
new file mode 100644
index 0000000000000000000000000000000000000000..1e6198a15f07d743cb4f2fc0466761e4d336fb12
GIT binary patch
literal 225
zcmV<703QDziwFP!32ul0|7^~&PQx$|2H=PV2`1hkItivI+&T6+WP^nTSa<?AcEw34
zjZ#-asrv5NqG71ISoZ1v{asNM53ds6+^T%p%izoBDazXQaaX>F=)8-r#2RaLlag15
zV>++L{c4c@F<tKuu-ST9OYqQ<HmYT6w5swnKnK<sIOgWvsUzP${M27@Go<0iSsN7^
zMy#NpdgvNuxv|<bRKZT>^qsYB1+$D}&TFOr8P@ubxjiv+n6~Npc?sZl0b0Z-CMH|}
bBE>o{upmTe&y?Q)00960N7SRnF#!Mo?;mgM

literal 0
HcmV?d00001

diff --git a/spec/fixtures/pages_empty.zip b/spec/fixtures/pages_empty.zip
new file mode 100644
index 0000000000000000000000000000000000000000..db3f0334c12d868859a979dc3f8fa3d8fa283813
GIT binary patch
literal 160
zcmWIWW@h1H00F@?eGf1LO0Y7>Fu3{#ghYmha56AYx-%mTgi9;985mi<Gcqu+hyWD^
ncr!A|G2=2o0&d=tMi3Kf94o{)45L`tK#CZF&>KkGgE$NTEA$na

literal 0
HcmV?d00001

diff --git a/spec/fixtures/pages_empty.zip.meta b/spec/fixtures/pages_empty.zip.meta
new file mode 100644
index 0000000000000000000000000000000000000000..d0b93b3b9c0fd3a1279532e9aa03b6f6915c2963
GIT binary patch
literal 116
zcmV-)0E_=0iwFP!32ul0|6*WZP;<{L@kvZla4OBrNl|btD#=VsOfD%_@J%gAOi3(B
zR4~vp&@<u!s${CJ1(K|;z5yYTK!$3yQf_`qW?E)yijtL~iK&r=nUSHPi4I6KRmn=p
W%)~^gmJ1L70RR7A4_dim0001m<1d~7

literal 0
HcmV?d00001

diff --git a/spec/services/projects/update_pages_worker_spec.rb b/spec/services/projects/update_pages_worker_spec.rb
index 0607c025b9e..68e66866340 100644
--- a/spec/services/projects/update_pages_worker_spec.rb
+++ b/spec/services/projects/update_pages_worker_spec.rb
@@ -4,9 +4,7 @@ describe Projects::UpdatePagesService do
   let(:project) { create :project }
   let(:commit) { create :ci_commit, project: project, sha: project.commit('HEAD').sha }
   let(:build) { create :ci_build, commit: commit, ref: 'HEAD' }
-  let(:file) { fixture_file_upload(Rails.root + 'spec/fixtures/pages.tar.gz', 'application/octet-stream') }
-  let(:empty_file) { fixture_file_upload(Rails.root + 'spec/fixtures/pages_empty.tar.gz', 'application/octet-stream') }
-  let(:invalid_file) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'application/octet-stream') }
+  let(:invalid_file) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png') }
   
   subject { described_class.new(project, build) }
 
@@ -14,27 +12,50 @@ describe Projects::UpdatePagesService do
     project.remove_pages
   end
 
-  context 'for valid file' do
-    before { build.update_attributes(artifacts_file: file) }
+  %w(tar.gz zip).each do |format|
+    context "for valid #{format}" do
+      let(:file) { fixture_file_upload(Rails.root + "spec/fixtures/pages.#{format}") }
+      let(:empty_file) { fixture_file_upload(Rails.root + "spec/fixtures/pages_empty.#{format}") }
+      let(:metadata) do
+        filename = Rails.root + "spec/fixtures/pages.#{format}.meta"
+        fixture_file_upload(filename) if File.exists?(filename)
+      end
 
-    it 'succeeds' do
-      expect(project.pages_url).to be_nil
-      expect(execute).to eq(:success)
-      expect(project.pages_url).to_not be_nil
-    end
+      before do
+        build.update_attributes(artifacts_file: file)
+        build.update_attributes(artifacts_metadata: metadata)
+      end
 
-    it 'limits pages size' do
-      stub_application_setting(max_pages_size: 1)
-      expect(execute).to_not eq(:success)
-    end
+      it 'succeeds' do
+        expect(project.pages_url).to be_nil
+        expect(execute).to eq(:success)
+        expect(project.pages_url).to_not be_nil
+      end
 
-    it 'removes pages after destroy' do
-      expect(PagesWorker).to receive(:perform_in)
-      expect(project.pages_url).to be_nil
-      expect(execute).to eq(:success)
-      expect(project.pages_url).to_not be_nil
-      project.destroy
-      expect(Dir.exist?(project.public_pages_path)).to be_falsey
+      it 'limits pages size' do
+        stub_application_setting(max_pages_size: 1)
+        expect(execute).to_not eq(:success)
+      end
+
+      it 'removes pages after destroy' do
+        expect(PagesWorker).to receive(:perform_in)
+        expect(project.pages_url).to be_nil
+        expect(execute).to eq(:success)
+        expect(project.pages_url).to_not be_nil
+        project.destroy
+        expect(Dir.exist?(project.public_pages_path)).to be_falsey
+      end
+
+      it 'fails if sha on branch is not latest' do
+        commit.update_attributes(sha: 'old_sha')
+        build.update_attributes(artifacts_file: file)
+        expect(execute).to_not eq(:success)
+      end
+
+      it 'fails for empty file fails' do
+        build.update_attributes(artifacts_file: empty_file)
+        expect(execute).to_not eq(:success)
+      end
     end
   end
 
@@ -48,21 +69,10 @@ describe Projects::UpdatePagesService do
     expect(execute).to_not eq(:success)
   end
 
-  it 'fails for empty file fails' do
-    build.update_attributes(artifacts_file: empty_file)
-    expect(execute).to_not eq(:success)
-  end
-
   it 'fails for invalid archive' do
     build.update_attributes(artifacts_file: invalid_file)
     expect(execute).to_not eq(:success)
   end
-
-  it 'fails if sha on branch is not latest' do
-    commit.update_attributes(sha: 'old_sha')
-    build.update_attributes(artifacts_file: file)
-    expect(execute).to_not eq(:success)
-  end
   
   def execute
     subject.execute[:status]