From a29356023704f63d9b80219067144a0a0247e927 Mon Sep 17 00:00:00 2001 From: Phillip Webb Date: Thu, 14 Nov 2024 13:43:28 -0800 Subject: [PATCH] Support nested OCI indexes Update `ExportedImageTar.IndexLayerArchiveFactory` to support nested indexes. Nested indexes support a layer of interaction where the `index.json` file points to a blob that contains the read index to use. Prior to this commit, we only supported indexes provided directly by the `index.json` file. This missing support results in "buildpack.toml: no such file or directory" errors when referencing specific buildpacks and using Docker Engine 27.3.1 or above. See gh-43126 --- .../platform/docker/ExportedImageTar.java | 50 +++++++++++++----- .../docker/ExportedImageTarTests.java | 10 +++- .../export-docker-desktop-nested-index.tar | Bin 0 -> 18944 bytes 3 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/export-docker-desktop-nested-index.tar diff --git a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java index 53048215980..ccbd2c0a833 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTar.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; @@ -142,23 +143,40 @@ class ExportedImageTar implements Closeable { private final Map layerMediaTypes; IndexLayerArchiveFactory(Path tarFile, ImageArchiveIndex index) throws IOException { - Set manifestDigests = getDigests(index, this::isManifest); - List manifestLists = getManifestLists(tarFile, getDigests(index, this::isManifestList)); + this(tarFile, withNestedIndexes(tarFile, index)); + } + + IndexLayerArchiveFactory(Path tarFile, List indexes) throws IOException { + Set manifestDigests = getDigests(indexes, this::isManifest); + Set manifestListDigests = getDigests(indexes, IndexLayerArchiveFactory::isManifestList); + List manifestLists = getManifestLists(tarFile, manifestListDigests); List manifests = getManifests(tarFile, manifestDigests, manifestLists); this.layerMediaTypes = manifests.stream() .flatMap((manifest) -> manifest.getLayers().stream()) - .collect(Collectors.toMap(this::getEntryName, BlobReference::getMediaType)); + .collect(Collectors.toMap(IndexLayerArchiveFactory::getEntryName, BlobReference::getMediaType)); } - private Set getDigests(ImageArchiveIndex index, Predicate predicate) { - return index.getManifests() - .stream() + private static List withNestedIndexes(Path tarFile, ImageArchiveIndex index) + throws IOException { + Set indexDigests = getDigests(Stream.of(index), IndexLayerArchiveFactory::isIndex); + List indexes = new ArrayList<>(); + indexes.add(index); + indexes.addAll(getDigestMatches(tarFile, indexDigests, ImageArchiveIndex::of)); + return indexes; + } + + private static Set getDigests(List indexes, Predicate predicate) { + return getDigests(indexes.stream(), predicate); + } + + private static Set getDigests(Stream indexes, Predicate predicate) { + return indexes.flatMap((index) -> index.getManifests().stream()) .filter(predicate) .map(BlobReference::getDigest) .collect(Collectors.toUnmodifiableSet()); } - private List getManifestLists(Path tarFile, Set digests) throws IOException { + private static List getManifestLists(Path tarFile, Set digests) throws IOException { return getDigestMatches(tarFile, digests, ManifestList::of); } @@ -173,12 +191,14 @@ class ExportedImageTar implements Closeable { return getDigestMatches(tarFile, digests, Manifest::of); } - private List getDigestMatches(Path tarFile, Set digests, + private static List getDigestMatches(Path tarFile, Set digests, ThrowingFunction factory) throws IOException { if (digests.isEmpty()) { return Collections.emptyList(); } - Set names = digests.stream().map(this::getEntryName).collect(Collectors.toUnmodifiableSet()); + Set names = digests.stream() + .map(IndexLayerArchiveFactory::getEntryName) + .collect(Collectors.toUnmodifiableSet()); List result = new ArrayList<>(); try (TarArchiveInputStream tar = openTar(tarFile)) { TarArchiveEntry entry = tar.getNextTarEntry(); @@ -197,19 +217,23 @@ class ExportedImageTar implements Closeable { || isJsonWithPrefix(reference.getMediaType(), "application/vnd.docker.distribution.manifest.v"); } - private boolean isManifestList(BlobReference reference) { + private static boolean isIndex(BlobReference reference) { + return isJsonWithPrefix(reference.getMediaType(), "application/vnd.oci.image.index.v"); + } + + private static boolean isManifestList(BlobReference reference) { return isJsonWithPrefix(reference.getMediaType(), "application/vnd.docker.distribution.manifest.list.v"); } - private boolean isJsonWithPrefix(String mediaType, String prefix) { + private static boolean isJsonWithPrefix(String mediaType, String prefix) { return mediaType.startsWith(prefix) && mediaType.endsWith("+json"); } - private String getEntryName(BlobReference reference) { + private static String getEntryName(BlobReference reference) { return getEntryName(reference.getDigest()); } - private String getEntryName(String digest) { + private static String getEntryName(String digest) { return "blobs/" + digest.replace(':', '/'); } diff --git a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java index 253ed2cb03d..4d9ea93c470 100644 --- a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java +++ b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java @@ -16,6 +16,9 @@ package org.springframework.boot.buildpack.platform.docker; +import java.util.ArrayList; +import java.util.List; + import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -34,7 +37,8 @@ class ExportedImageTarTests { @ParameterizedTest @ValueSource(strings = { "export-docker-desktop.tar", "export-docker-desktop-containerd.tar", - "export-docker-desktop-containerd-manifest-list.tar", "export-docker-engine.tar", "export-podman.tar" }) + "export-docker-desktop-containerd-manifest-list.tar", "export-docker-engine.tar", "export-podman.tar", + "export-docker-desktop-nested-index.tar" }) void test(String tarFile) throws Exception { ImageReference reference = ImageReference.of("test:latest"); try (ExportedImageTar exportedImageTar = new ExportedImageTar(reference, @@ -43,10 +47,12 @@ class ExportedImageTarTests { String expectedName = (expectedCompression != Compression.GZIP) ? "5caae51697b248b905dca1a4160864b0e1a15c300981736555cdce6567e8d477" : "f0f1fd1bdc71ac6a4dc99cea5f5e45c86c5ec26fe4d1daceeb78207303606429"; + List names = new ArrayList<>(); exportedImageTar.exportLayers((name, tarArchive) -> { - assertThat(name).contains(expectedName); + names.add(name); assertThat(tarArchive.getCompression()).isEqualTo(expectedCompression); }); + assertThat(names).filteredOn((name) -> name.contains(expectedName)).isNotEmpty(); } } diff --git a/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/export-docker-desktop-nested-index.tar b/spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/export-docker-desktop-nested-index.tar new file mode 100644 index 0000000000000000000000000000000000000000..bf423d69ba2f89ca6e608385bfd9c90789dc362b GIT binary patch literal 18944 zcmeGj{ZHFUlArxo>~kx%eMg-2`a7?!L?QG=M*)?zCmkTP_O8jkCU(wtXgQSs{bp^) z`Dh4)0FNHAkl5_UcxHCzV`s*GH}=!|#)Cc>W5=@4hWu|A|1mN1-_Vsm2oqu(7BMi^ z5!Q*RlMQ4&C{`u#L7LG7Q0sr^zQ5pwFf9xhjF)ssBH=*`WFAbe8@7RA{|(bz@BgEmld1i8Ov5!C=D9BCo<*>QLmTU+W$GS=5tkBXTfXT8 zJPbWzdY13GEbzE#c^n4;z6N91f7>kNeXji5wu!3!Cm{dCux)DtT?g-)*W9i9nalr` zng$)-qhEQFia1gmWK->NCTRPz&((%X`~9v6Xa-~Ti->7)AT-gV=UfvJ<3F?u{U3j& zuntzyNQ6AiQnhh>bxRndH4|W%I0qo`=0YbmC=M;dr-YcHZ@Qk%Lc=i3Adu%Sw83gn zOyGc;#AlvvIXJMG&s@v3b<41cPXV)3{1-?uO?y+NQ50u6HGt-oDq<~&ql}7(C#<9^ zqCH7?KZ*b1L6&L>KNo41Txu7+?=<1CpjS`(iHOeqIL_)BU|SPW7&p39w&8toa<0XF z9#x>z5_G~ttps0A9&W0AXohjpllagi=!gtnJ4kZo_n2);fPk#p717{_Ivk!3?{?7V zubPs7E1;BHy6rhWFZTgt&RJQ`K(VnY2hHY6EYy-C~+u<%V9cTV4 zq)A*L|GFjZ<&^y=hC@LAujT(5hTda$J&63eKM-Bkr@^uJh;u`84e)DbG-i$EYv*JfxzlC_L!INUhas?@=3kT+{-87gb(E z(E^89Rg@Q~wSV}fjkZ32{PL-}-#$`C{_PnP$-9?Cf!7qxtI3g}e7yo_Fo2WJoiG|y z6cM81V^j;#%c~OFu)cfndAoVGyVctJ^0|plPyaw!herxR1@60X5^z>v$`d5Xl0LQ^ z{~NbJJ%e}IJA6~0A}1UX0}V+^Nz4RI#U^}ewzq1n{oVSft^L++^Qf(T+HD|pyr1(X zOztH@a1}-KX62l_;h0JetIE@q!@mowW%WPf@_M=)wo)%H@c(t}=rj6X#|HTSk9z-G zj-;PwOCsU<|J464kglQaKbzbCM6F%4b$Gbf+TQxG*DNLXH>r!~N#(Tc7Md30TvlHo zqM?t?+*0I5B0B5StRpQ17!gB5JI(FAEl}k<&BNyYPIG^|)jUG44-VU{gZ-_&H;U8* z@83%;4R)khJXF!!x6Ok+0PtaSr9}|Pxe$z}U=Ku!7>B?NgV@Za@&cyqAGDhd)b8+! z0R+>^MGUcljPyubf?nkY@0A=2c$#Ve;P`k1{q$7Hb^AM{R@dg-9F7r5_S%^sU zf+v@NC%loSfHVq|xQAXz$gdA?{vqAisFzk}+Qjx<<&(rwe%evjnVrenAc_rT<4VrDbuidwXJ~UW5Q( zr5fdLct*dm3BmrU_z%$?Vz2f8?Kyn+nUk7w>#&7D0m(doLi+-NF2fp`(PFI~L?ecJ zJfn+sjcxFP@ae?w9lZc|yDT;|#S8zv7=r*hbvY67M{ z%ro#=QD)Gkm*98+R3^gkOfXqQFscApH7l*2LY5tJ@u4`EIrcs3^X@47U%3LQRyDN} znTC3GGT)gw0XQe%8-W*7G{IYfhU_9I>L#qp-3aJ9hIxYJV{?6p6BT%KqULYh0n%GL z`P{PhVO7d=Tt?1Q4tP25DPJ!ypZVGw+%zC%ID{PpIxALAa8;I!jGRU|RAau(br+X> zGqA99FLO-Goq^s}iC=0Xhmr%==)kAiJ*EFwt+ly^$Q$%OwjCSE3t{<4{jbaV&qs~_ zJzM>M*ZQwHwOeB)d%F^{V*Y~SuT>oqP)9}amuq^|Z{_fiQqCI32yk}h!MmX`l7~3|arNmj7!QdXL>u11WtxFNE@4J+P?GbttRkCNnW5 zI(1x+Q0$nT8a^{TDAINvLVe2&%)qxoWDoQ?2^q0o04mcJc(4mYwYk+|;st0P+86 zIQg%GC;%kNInC%oZOBCLrn=n&_7u&QEPE}``5JczJr0VY82J}OkRaN=BT8(W5!8$J zV-W#f#f*dKgAAUIevW)lIXLB5)aNZ<2p(IJ5Fkpd@g8|6jz^dm4iqm%P@3h`|x~ zI{F9{sX8P#u==Y-v+s#1&j?HcIyx1Q$P+w$D6(&oe}c_f`FF@V{`XMZF$MQ(`463? iaVtMxZm268;o)%BBu(MB{;}r3ngeSNtU2&79QYr9%}db$ literal 0 HcmV?d00001