From a29356023704f63d9b80219067144a0a0247e927 Mon Sep 17 00:00:00 2001 From: Phillip Webb Date: Thu, 14 Nov 2024 13:43:28 -0800 Subject: [PATCH 1/2] Support nested OCI indexes Update `ExportedImageTar.IndexLayerArchiveFactory` to support nested indexes. Nested indexes support a layer of interaction where the `index.json` file points to a blob that contains the read index to use. Prior to this commit, we only supported indexes provided directly by the `index.json` file. This missing support results in "buildpack.toml: no such file or directory" errors when referencing specific buildpacks and using Docker Engine 27.3.1 or above. See gh-43126 --- .../platform/docker/ExportedImageTar.java | 50 +++++++++++++----- .../docker/ExportedImageTarTests.java | 10 +++- .../export-docker-desktop-nested-index.tar | Bin 0 -> 18944 bytes 3 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/export-docker-desktop-nested-index.tar diff --git a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java index 53048215980..ccbd2c0a833 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; @@ -142,23 +143,40 @@ class ExportedImageTar implements Closeable { private final Map layerMediaTypes; IndexLayerArchiveFactory(Path tarFile, ImageArchiveIndex index) throws IOException { - Set manifestDigests = getDigests(index, this::isManifest); - List manifestLists = getManifestLists(tarFile, getDigests(index, this::isManifestList)); + this(tarFile, withNestedIndexes(tarFile, index)); + } + + IndexLayerArchiveFactory(Path tarFile, List indexes) throws IOException { + Set manifestDigests = getDigests(indexes, this::isManifest); + Set manifestListDigests = getDigests(indexes, IndexLayerArchiveFactory::isManifestList); + List manifestLists = getManifestLists(tarFile, manifestListDigests); List manifests = getManifests(tarFile, manifestDigests, manifestLists); this.layerMediaTypes = manifests.stream() .flatMap((manifest) -> manifest.getLayers().stream()) - .collect(Collectors.toMap(this::getEntryName, BlobReference::getMediaType)); + .collect(Collectors.toMap(IndexLayerArchiveFactory::getEntryName, BlobReference::getMediaType)); } - private Set getDigests(ImageArchiveIndex index, Predicate predicate) { - return index.getManifests() - .stream() + private static List withNestedIndexes(Path tarFile, ImageArchiveIndex index) + throws IOException { + Set indexDigests = getDigests(Stream.of(index), IndexLayerArchiveFactory::isIndex); + List indexes = new ArrayList<>(); + indexes.add(index); + indexes.addAll(getDigestMatches(tarFile, indexDigests, ImageArchiveIndex::of)); + return indexes; + } + + private static Set getDigests(List indexes, Predicate predicate) { + return getDigests(indexes.stream(), predicate); + } + + private static Set getDigests(Stream indexes, Predicate predicate) { + return indexes.flatMap((index) -> index.getManifests().stream()) .filter(predicate) .map(BlobReference::getDigest) .collect(Collectors.toUnmodifiableSet()); } - private List getManifestLists(Path tarFile, Set digests) throws IOException { + private static List getManifestLists(Path tarFile, Set digests) throws IOException { return getDigestMatches(tarFile, digests, ManifestList::of); } @@ -173,12 +191,14 @@ class ExportedImageTar implements Closeable { return getDigestMatches(tarFile, digests, Manifest::of); } - private List getDigestMatches(Path tarFile, Set digests, + private static List getDigestMatches(Path tarFile, Set digests, ThrowingFunction factory) throws IOException { if (digests.isEmpty()) { return Collections.emptyList(); } - Set names = digests.stream().map(this::getEntryName).collect(Collectors.toUnmodifiableSet()); + Set names = digests.stream() + .map(IndexLayerArchiveFactory::getEntryName) + .collect(Collectors.toUnmodifiableSet()); List result = new ArrayList<>(); try (TarArchiveInputStream tar = openTar(tarFile)) { TarArchiveEntry entry = tar.getNextTarEntry(); @@ -197,19 +217,23 @@ class ExportedImageTar implements Closeable { || isJsonWithPrefix(reference.getMediaType(), "application/vnd.docker.distribution.manifest.v"); } - private boolean isManifestList(BlobReference reference) { + private static boolean isIndex(BlobReference reference) { + return isJsonWithPrefix(reference.getMediaType(), "application/vnd.oci.image.index.v"); + } + + private static boolean isManifestList(BlobReference reference) { return isJsonWithPrefix(reference.getMediaType(), "application/vnd.docker.distribution.manifest.list.v"); } - private boolean isJsonWithPrefix(String mediaType, String prefix) { + private static boolean isJsonWithPrefix(String mediaType, String prefix) { return mediaType.startsWith(prefix) && mediaType.endsWith("+json"); } - private String getEntryName(BlobReference reference) { + private static String getEntryName(BlobReference reference) { return getEntryName(reference.getDigest()); } - private String getEntryName(String digest) { + private static String getEntryName(String digest) { return "blobs/" + digest.replace(':', '/'); } diff --git a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java index 253ed2cb03d..4d9ea93c470 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java @@ -16,6 +16,9 @@ package org.springframework.boot.buildpack.platform.docker; +import java.util.ArrayList; +import java.util.List; + import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -34,7 +37,8 @@ class ExportedImageTarTests { @ParameterizedTest @ValueSource(strings = { "export-docker-desktop.tar", "export-docker-desktop-containerd.tar", - "export-docker-desktop-containerd-manifest-list.tar", "export-docker-engine.tar", "export-podman.tar" }) + "export-docker-desktop-containerd-manifest-list.tar", "export-docker-engine.tar", "export-podman.tar", + "export-docker-desktop-nested-index.tar" }) void test(String tarFile) throws Exception { ImageReference reference = ImageReference.of("test:latest"); try (ExportedImageTar exportedImageTar = new ExportedImageTar(reference, @@ -43,10 +47,12 @@ class ExportedImageTarTests { String expectedName = (expectedCompression != Compression.GZIP) ? "5caae51697b248b905dca1a4160864b0e1a15c300981736555cdce6567e8d477" : "f0f1fd1bdc71ac6a4dc99cea5f5e45c86c5ec26fe4d1daceeb78207303606429"; + List names = new ArrayList<>(); exportedImageTar.exportLayers((name, tarArchive) -> { - assertThat(name).contains(expectedName); + names.add(name); assertThat(tarArchive.getCompression()).isEqualTo(expectedCompression); }); + assertThat(names).filteredOn((name) -> name.contains(expectedName)).isNotEmpty(); } } diff --git a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/export-docker-desktop-nested-index.tar b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/export-docker-desktop-nested-index.tar new file mode 100644 index 0000000000000000000000000000000000000000..bf423d69ba2f89ca6e608385bfd9c90789dc362b GIT binary patch literal 18944 zcmeGj{ZHFUlArxo>~kx%eMg-2`a7?!L?QG=M*)?zCmkTP_O8jkCU(wtXgQSs{bp^) z`Dh4)0FNHAkl5_UcxHCzV`s*GH}=!|#)Cc>W5=@4hWu|A|1mN1-_Vsm2oqu(7BMi^ z5!Q*RlMQ4&C{`u#L7LG7Q0sr^zQ5pwFf9xhjF)ssBH=*`WFAbe8@7RA{|(bz@BgEmld1i8Ov5!C=D9BCo<*>QLmTU+W$GS=5tkBXTfXT8 zJPbWzdY13GEbzE#c^n4;z6N91f7>kNeXji5wu!3!Cm{dCux)DtT?g-)*W9i9nalr` zng$)-qhEQFia1gmWK->NCTRPz&((%X`~9v6Xa-~Ti->7)AT-gV=UfvJ<3F?u{U3j& zuntzyNQ6AiQnhh>bxRndH4|W%I0qo`=0YbmC=M;dr-YcHZ@Qk%Lc=i3Adu%Sw83gn zOyGc;#AlvvIXJMG&s@v3b<41cPXV)3{1-?uO?y+NQ50u6HGt-oDq<~&ql}7(C#<9^ zqCH7?KZ*b1L6&L>KNo41Txu7+?=<1CpjS`(iHOeqIL_)BU|SPW7&p39w&8toa<0XF z9#x>z5_G~ttps0A9&W0AXohjpllagi=!gtnJ4kZo_n2);fPk#p717{_Ivk!3?{?7V zubPs7E1;BHy6rhWFZTgt&RJQ`K(VnY2hHY6EYy-C~+u<%V9cTV4 zq)A*L|GFjZ<&^y=hC@LAujT(5hTda$J&63eKM-Bkr@^uJh;u`84e)DbG-i$EYv*JfxzlC_L!INUhas?@=3kT+{-87gb(E z(E^89Rg@Q~wSV}fjkZ32{PL-}-#$`C{_PnP$-9?Cf!7qxtI3g}e7yo_Fo2WJoiG|y z6cM81V^j;#%c~OFu)cfndAoVGyVctJ^0|plPyaw!herxR1@60X5^z>v$`d5Xl0LQ^ z{~NbJJ%e}IJA6~0A}1UX0}V+^Nz4RI#U^}ewzq1n{oVSft^L++^Qf(T+HD|pyr1(X zOztH@a1}-KX62l_;h0JetIE@q!@mowW%WPf@_M=)wo)%H@c(t}=rj6X#|HTSk9z-G zj-;PwOCsU<|J464kglQaKbzbCM6F%4b$Gbf+TQxG*DNLXH>r!~N#(Tc7Md30TvlHo zqM?t?+*0I5B0B5StRpQ17!gB5JI(FAEl}k<&BNyYPIG^|)jUG44-VU{gZ-_&H;U8* z@83%;4R)khJXF!!x6Ok+0PtaSr9}|Pxe$z}U=Ku!7>B?NgV@Za@&cyqAGDhd)b8+! z0R+>^MGUcljPyubf?nkY@0A=2c$#Ve;P`k1{q$7Hb^AM{R@dg-9F7r5_S%^sU zf+v@NC%loSfHVq|xQAXz$gdA?{vqAisFzk}+Qjx<<&(rwe%evjnVrenAc_rT<4VrDbuidwXJ~UW5Q( zr5fdLct*dm3BmrU_z%$?Vz2f8?Kyn+nUk7w>#&7D0m(doLi+-NF2fp`(PFI~L?ecJ zJfn+sjcxFP@ae?w9lZc|yDT;|#S8zv7=r*hbvY67M{ z%ro#=QD)Gkm*98+R3^gkOfXqQFscApH7l*2LY5tJ@u4`EIrcs3^X@47U%3LQRyDN} znTC3GGT)gw0XQe%8-W*7G{IYfhU_9I>L#qp-3aJ9hIxYJV{?6p6BT%KqULYh0n%GL z`P{PhVO7d=Tt?1Q4tP25DPJ!ypZVGw+%zC%ID{PpIxALAa8;I!jGRU|RAau(br+X> zGqA99FLO-Goq^s}iC=0Xhmr%==)kAiJ*EFwt+ly^$Q$%OwjCSE3t{<4{jbaV&qs~_ zJzM>M*ZQwHwOeB)d%F^{V*Y~SuT>oqP)9}amuq^|Z{_fiQqCI32yk}h!MmX`l7~3|arNmj7!QdXL>u11WtxFNE@4J+P?GbttRkCNnW5 zI(1x+Q0$nT8a^{TDAINvLVe2&%)qxoWDoQ?2^q0o04mcJc(4mYwYk+|;st0P+86 zIQg%GC;%kNInC%oZOBCLrn=n&_7u&QEPE}``5JczJr0VY82J}OkRaN=BT8(W5!8$J zV-W#f#f*dKgAAUIevW)lIXLB5)aNZ<2p(IJ5Fkpd@g8|6jz^dm4iqm%P@3h`|x~ zI{F9{sX8P#u==Y-v+s#1&j?HcIyx1Q$P+w$D6(&oe}c_f`FF@V{`XMZF$MQ(`463? iaVtMxZm268;o)%BBu(MB{;}r3ngeSNtU2&79QYr9%}db$ literal 0 HcmV?d00001 From 25b6477aa84cf18794e40c7df73dd87f52c52508 Mon Sep 17 00:00:00 2001 From: Phillip Webb Date: Thu, 14 Nov 2024 15:21:37 -0800 Subject: [PATCH 2/2] Support alternative media type format Update `ExportedImageTar` media type detection to support `tar+gzip` as well as `tar.gzip`. Recent updates to Docker Desktop appear to have changed the format. Fixes gh-43126 --- .../platform/docker/ExportedImageTar.java | 4 ++-- .../platform/docker/ExportedImageTarTests.java | 2 +- ...-docker-desktop-containerd-alt-mediatype.tar | Bin 0 -> 19968 bytes 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/export-docker-desktop-containerd-alt-mediatype.tar diff --git a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java index ccbd2c0a833..9e7daa38ef3 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java @@ -247,10 +247,10 @@ class ExportedImageTar implements Closeable { } private Compression getCompression(String mediaType) { - if (mediaType.endsWith(".tar.gzip")) { + if (mediaType.endsWith(".tar.gzip") || mediaType.endsWith(".tar+gzip")) { return Compression.GZIP; } - if (mediaType.endsWith(".tar.zstd")) { + if (mediaType.endsWith(".tar.zstd") || mediaType.endsWith(".tar+zstd")) { return Compression.ZSTD; } return Compression.NONE; diff --git a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java index 4d9ea93c470..6ba75321e88 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java @@ -38,7 +38,7 @@ class ExportedImageTarTests { @ParameterizedTest @ValueSource(strings = { "export-docker-desktop.tar", "export-docker-desktop-containerd.tar", "export-docker-desktop-containerd-manifest-list.tar", "export-docker-engine.tar", "export-podman.tar", - "export-docker-desktop-nested-index.tar" }) + "export-docker-desktop-nested-index.tar", "export-docker-desktop-containerd-alt-mediatype.tar" }) void test(String tarFile) throws Exception { ImageReference reference = ImageReference.of("test:latest"); try (ExportedImageTar exportedImageTar = new ExportedImageTar(reference, diff --git a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/export-docker-desktop-containerd-alt-mediatype.tar b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/export-docker-desktop-containerd-alt-mediatype.tar new file mode 100644 index 0000000000000000000000000000000000000000..71be13d0a03858a83bca12eeb5fb5cc79dbae395 GIT binary patch literal 19968 zcmeGk3v?9KHKHh{pnQT*v}7}ifF!#!v-8;vKMfJYB3Mj>l0b0P^f?);Uo9~j8;A(8pR*A21F=SS_2|Ngv4y`%w&N?cS+b}mBY;0lbN~i zz5DLH@7?>}yZ25Uxw6A&iD}ad$8naXO)=`7(eF4(Y3~5iE>rLU?k1cdtqd7sqT7_J z2IP4@1$luqPrl%Agy|5fiDH=Y20m0K;WCA(`HL!YXyfn8MkLL&)ieNkv~c{bR@NB* zsLYc_<1gYOA@YQS=U4)945E0>Zs!CgVfg>4Fpm>5uoFF(@8;P@4+{)k# zMcPA94C7Bxy1xIE{u3+>I)C9ZfpK80Y>WvHfv;7;SGoSjKiNSSXW;8`t}%z+=}~j~ zvEq$8QuojC$Go{cz1#Lf6=!enykp#)(xr?4^=gJV)%o~wdgqC0)}*=jS7q(JwPuLt zfb!j-A4b1#ujoJj{S}o;cEJ%um2(c`FR!ncinpKi(GeYAYlUv_&)07t6Lt}G<Bs%}#utyF} z-x^prrZRK)noT7u@IH5?6i-??_L>`in|R&eS$wq-xq zuZtsL-mV8bp_<8Es&^l|Y3;z{Sua1>}S?KHKdbwdD^W0Ip$Tu zchm7sblir-wH9&r;3E%iyKZ&d4;$M%4_-&*eLraaGwX*XB=qUp(=zOAUHf)3V~VRv zR_t20WoVybFS%@8ZE4rDis!X0(>K_9-8S>!cS78em%km>?^kc#R{7k%t_z5cWzUvu zYJYU@q)z!?dJYb%%^x4v?dabVi~E04a>F+7i0T*iok$*6{rbK z)pUKM@40(dkNxwB`K#{XmQLI@dAjAZlEAeutSV>soV3>O8nKD4FaL5lwasqNveIQ%NRZfMW>TkomZKk)9yZ>+5BIMlqjEYW74e8b?h znWv6?+IQ^o#3ixH_PFb)rGu7KO~1Q)+RXY#|K0noet+xJ9#6Nm@BHq2LrQx+YROnI zc-g5{W0@D0oP6bvL!MnSuj80EC#-$I|C_Zl?mJhzXm@y8{Qm9z?_Id) z8|yRO7j0eJ?(02yug*SD-n;6>x;`7a1bT>@Z`t_$X?w@`%{Sd#QCoXr-6xs<2`rnM zqY6owHR^)o@SFzsqxSaCbg83gzSwns_@^`V_{3)-GP5L?$?&3{DXOLSVv4&dyT|L6f-E zW@AW}7X_MNaXZJbcAT;bA-cJg{ijKyiT$^-lwtovNV9dpg&6X3*^(k~N}gB1k}>4s z8463pIIn;dfd>d-x8X@R0lzdHPgZ}#1gMZL`4rh(fF)-XUaE3YRw5?*09CH!&YJ>l zy|S!`04T(i0*~5jE-DbbJ}BWOQJg68P?mw|yV7KuLkQ*>V`m*CWpmhZn&%LKD43Ww zhH~Hnfe4zj;<(*LuvUhqX^!UvhGtm7##1bdWfcKCvRjm#YLBCkL&(+omSnRdPs-&z zh|BR^cr-TqKoq8tAPQ4TP=dfp zQ4x$_2&n=EW|yEK9w`V0{DNFUNEoEj{GOq*ALfrFyEQHwMh2jfGT)+sEK()Vy6V~H z*c#b?&~@{FAl7{sxx9f`P2w9yR7&@7x5><+|ppo_G? zhM*SgKND{Mj}w%Mpjd*Yagt>wmG|fVrFy!&%N*q!^$t-?1Z{}q#NAQ|?SO&b3BTt2!UC1qoU>eMRa=;{??8bCF zLE~%EdQ&@W4rXYjAju7VNG{cS_=$^fYE(;8VeK&4D7)5;3gRUvbPvGb;t}bV22%x5 zq_YuR7fcx{5UwCQ@YpPlbUyUO4+JNQX7D`?a88vxx}Y?qK)TKs0eTZ;Ozq)EyhVCk z*LVZWMJhX4u~(YuYE?bBT9FA;JhE3w(v>-h7koL2>`6MWdb~PABb7~f`%l8o4L@)H zVG#iP4<`=7u>X2jn8cP?LYB_?Rr-iZq5AV`!`~AP!*#G}LQ{U(nB+$tIM|xNeAyL&d zhO__R|8HXd2`1|PpAfV+8+@hOf5ZQ8`2W=ihvEPKso=-e<^PY=p%}tgUdsMM{!bJ8 z56NiJ_Wy^Vz1iR^)cza!KMxrBKSut~Rh$1Kxp`rVS&h{-+tRhTqG}o;035XOP476w zk%KU`Y79;gBoXcWpB8h~q=8B~TtP;fl>TC03!BD%$wVv|#Z{AV3NH?9USqt^vyzbhYk z1vMQ8Qd9JVl*So9dS1)LsI`!^nxO(M!Xa$Pr)i8XITv!PAWjWwQC_E6_6Y9tsC^C5 zZm%Gk8_+jYE-K1u{Sa+%tlQEAsCp4j|E=tg^dFKz4F7L5j0Pz9u!ht)tW|wMFt^7F z9ytv;wOqc-RuM=Y9Hl1*LPlUC3L`zOKwe;0G(`N1?nCCJB*<%#^OUwDQ`P@4{y!W( z9|R070w6 z>OY+So9aKy8u`DMu-vUvY@GjFM~+u2mMES7;rJis|G!w7rut7ZkpCOi^Z!-5{u}52 TD^+(5x(x{!5-=nXi3I)++QIuU literal 0 HcmV?d00001