Update vendor of containers/(common,image)
Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>
This commit is contained in:
parent
d6e2dd76c4
commit
bbea3eb544
3
Makefile
3
Makefile
|
@ -14,6 +14,8 @@ BINDIR := $(PREFIX)/bin
|
|||
BASHINSTALLDIR = $(PREFIX)/share/bash-completion/completions
|
||||
BUILDFLAGS := -tags "$(BUILDTAGS)"
|
||||
BUILDAH := buildah
|
||||
SELINUXOPT ?= $(shell test -x /usr/sbin/selinuxenabled && selinuxenabled && echo -Z)
|
||||
SELINUXTYPE=container_runtime_exec_t
|
||||
|
||||
GO := go
|
||||
GO_LDFLAGS := $(shell if $(GO) version|grep -q gccgo; then echo "-gccgoflags"; else echo "-ldflags"; fi)
|
||||
|
@ -75,6 +77,7 @@ static:
|
|||
|
||||
bin/buildah: $(SOURCES) cmd/buildah/*.go internal/mkcw/embed/entrypoint_amd64.gz
|
||||
$(GO_BUILD) $(BUILDAH_LDFLAGS) $(GO_GCFLAGS) "$(GOGCFLAGS)" -o $@ $(BUILDFLAGS) ./cmd/buildah
|
||||
test -z "${SELINUXOPT}" || chcon --verbose -t $(SELINUXTYPE) $@
|
||||
|
||||
ifneq ($(shell as --version | grep x86_64),)
|
||||
internal/mkcw/embed/entrypoint_amd64.gz: internal/mkcw/embed/entrypoint_amd64
|
||||
|
|
2
go.mod
2
go.mod
|
@ -6,7 +6,7 @@ require (
|
|||
github.com/containerd/containerd v1.7.13
|
||||
github.com/containernetworking/cni v1.1.2
|
||||
github.com/containernetworking/plugins v1.4.0
|
||||
github.com/containers/common v0.57.1-0.20240220203037-6ee157e78afb
|
||||
github.com/containers/common v0.57.1-0.20240301113114-0b996b05cd16
|
||||
github.com/containers/image/v5 v5.29.3-0.20240229213915-cdc68020a24f
|
||||
github.com/containers/luksy v0.0.0-20240212203526-ceb12d4fd50c
|
||||
github.com/containers/ocicrypt v1.1.9
|
||||
|
|
4
go.sum
4
go.sum
|
@ -58,8 +58,8 @@ github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl3
|
|||
github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
|
||||
github.com/containernetworking/plugins v1.4.0 h1:+w22VPYgk7nQHw7KT92lsRmuToHvb7wwSv9iTbXzzic=
|
||||
github.com/containernetworking/plugins v1.4.0/go.mod h1:UYhcOyjefnrQvKvmmyEKsUA+M9Nfn7tqULPpH0Pkcj0=
|
||||
github.com/containers/common v0.57.1-0.20240220203037-6ee157e78afb h1:i/kvuY7/Pu2hUgaJCqK4lbnVeXRCFDpvwJvfMv0i9Bs=
|
||||
github.com/containers/common v0.57.1-0.20240220203037-6ee157e78afb/go.mod h1:pc++5s/cvBtAkoAZ9d6czDqYer0yydX3d8LH6ALnwtA=
|
||||
github.com/containers/common v0.57.1-0.20240301113114-0b996b05cd16 h1:4Sv7nReBR3xgMecWnWt2ty6owlm32dDslPlGSYRdE08=
|
||||
github.com/containers/common v0.57.1-0.20240301113114-0b996b05cd16/go.mod h1:8irlyBcVooYx0F+YmoY7PQPAIgdJvCj17bvL7PqeaxI=
|
||||
github.com/containers/image/v5 v5.29.3-0.20240229213915-cdc68020a24f h1:DEK6PaY5/B6CYXjtdfAQGCUltHEPaoXvLb+C0PH6HiE=
|
||||
github.com/containers/image/v5 v5.29.3-0.20240229213915-cdc68020a24f/go.mod h1:a48d1rhHBl2zb630MSf20QQo4eIlIQvhZTqTcVJhbpA=
|
||||
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
|
||||
|
|
|
@ -18,6 +18,8 @@ const (
|
|||
EventTypeUnknown EventType = iota
|
||||
// EventTypeImagePull represents an image pull.
|
||||
EventTypeImagePull
|
||||
// EventTypeImagePullError represents an image pull failed.
|
||||
EventTypeImagePullError
|
||||
// EventTypeImagePush represents an image push.
|
||||
EventTypeImagePush
|
||||
// EventTypeImageRemove represents an image removal.
|
||||
|
@ -46,6 +48,8 @@ type Event struct {
|
|||
Time time.Time
|
||||
// Type of the event.
|
||||
Type EventType
|
||||
// Error in case of failure.
|
||||
Error error
|
||||
}
|
||||
|
||||
// writeEvent writes the specified event to the Runtime's event channel. The
|
||||
|
|
|
@ -53,13 +53,37 @@ type PullOptions struct {
|
|||
// The error is storage.ErrImageUnknown iff the pull policy is set to "never"
|
||||
// and no local image has been found. This allows for an easier integration
|
||||
// into some users of this package (e.g., Buildah).
|
||||
func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullPolicy, options *PullOptions) ([]*Image, error) {
|
||||
func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullPolicy, options *PullOptions) (_ []*Image, pullError error) {
|
||||
logrus.Debugf("Pulling image %s (policy: %s)", name, pullPolicy)
|
||||
|
||||
if r.eventChannel != nil {
|
||||
defer func() {
|
||||
if pullError != nil {
|
||||
// Note that we use the input name here to preserve the transport data.
|
||||
r.writeEvent(&Event{Name: name, Time: time.Now(), Type: EventTypeImagePullError, Error: pullError})
|
||||
}
|
||||
}()
|
||||
}
|
||||
if options == nil {
|
||||
options = &PullOptions{}
|
||||
}
|
||||
|
||||
defaultConfig, err := config.Default()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if options.MaxRetries == nil {
|
||||
options.MaxRetries = &defaultConfig.Engine.Retry
|
||||
}
|
||||
if options.RetryDelay == nil {
|
||||
if defaultConfig.Engine.RetryDelay != "" {
|
||||
duration, err := time.ParseDuration(defaultConfig.Engine.RetryDelay)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse containers.conf retry_delay: %w", err)
|
||||
}
|
||||
options.RetryDelay = &duration
|
||||
}
|
||||
}
|
||||
|
||||
var possiblyUnqualifiedName string // used for short-name resolution
|
||||
ref, err := alltransports.ParseImageName(name)
|
||||
if err != nil {
|
||||
|
@ -133,28 +157,25 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
|
|||
options.Variant = r.systemContext.VariantChoice
|
||||
}
|
||||
|
||||
var (
|
||||
pulledImages []string
|
||||
pullError error
|
||||
)
|
||||
var pulledImages []string
|
||||
|
||||
// Dispatch the copy operation.
|
||||
switch ref.Transport().Name() {
|
||||
// DOCKER REGISTRY
|
||||
case registryTransport.Transport.Name():
|
||||
pulledImages, pullError = r.copyFromRegistry(ctx, ref, possiblyUnqualifiedName, pullPolicy, options)
|
||||
pulledImages, err = r.copyFromRegistry(ctx, ref, possiblyUnqualifiedName, pullPolicy, options)
|
||||
|
||||
// DOCKER ARCHIVE
|
||||
case dockerArchiveTransport.Transport.Name():
|
||||
pulledImages, pullError = r.copyFromDockerArchive(ctx, ref, &options.CopyOptions)
|
||||
pulledImages, err = r.copyFromDockerArchive(ctx, ref, &options.CopyOptions)
|
||||
|
||||
// ALL OTHER TRANSPORTS
|
||||
default:
|
||||
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
|
||||
pulledImages, err = r.copyFromDefault(ctx, ref, &options.CopyOptions)
|
||||
}
|
||||
|
||||
if pullError != nil {
|
||||
return nil, pullError
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
localImages := []*Image{}
|
||||
|
|
|
@ -4,10 +4,14 @@ package libimage
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/containers/common/pkg/config"
|
||||
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
|
||||
dockerDaemonTransport "github.com/containers/image/v5/docker/daemon"
|
||||
"github.com/containers/image/v5/docker/reference"
|
||||
"github.com/containers/image/v5/manifest"
|
||||
"github.com/containers/image/v5/transports/alltransports"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
@ -31,6 +35,23 @@ func (r *Runtime) Push(ctx context.Context, source, destination string, options
|
|||
options = &PushOptions{}
|
||||
}
|
||||
|
||||
defaultConfig, err := config.Default()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if options.MaxRetries == nil {
|
||||
options.MaxRetries = &defaultConfig.Engine.Retry
|
||||
}
|
||||
if options.RetryDelay == nil {
|
||||
if defaultConfig.Engine.RetryDelay != "" {
|
||||
duration, err := time.ParseDuration(defaultConfig.Engine.RetryDelay)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse containers.conf retry_delay: %w", err)
|
||||
}
|
||||
options.RetryDelay = &duration
|
||||
}
|
||||
}
|
||||
|
||||
// Look up the local image. Note that we need to ignore the platform
|
||||
// and push what the user specified (containers/podman/issues/10344).
|
||||
image, resolvedSource, err := r.LookupImage(source, nil)
|
||||
|
@ -65,6 +86,14 @@ func (r *Runtime) Push(ctx context.Context, source, destination string, options
|
|||
destRef = dockerRef
|
||||
}
|
||||
|
||||
// docker-archive and only DockerV2Schema2MediaType support Gzip compression
|
||||
if options.CompressionFormat != nil &&
|
||||
(destRef.Transport().Name() == dockerArchiveTransport.Transport.Name() ||
|
||||
destRef.Transport().Name() == dockerDaemonTransport.Transport.Name() ||
|
||||
options.ManifestMIMEType == manifest.DockerV2Schema2MediaType) {
|
||||
options.CompressionFormat = nil
|
||||
}
|
||||
|
||||
if r.eventChannel != nil {
|
||||
defer r.writeEvent(&Event{ID: image.ID(), Name: destination, Time: time.Now(), Type: EventTypeImagePush})
|
||||
}
|
||||
|
|
79
vendor/github.com/containers/common/libnetwork/internal/rootlessnetns/netns_linux.go
generated
vendored
79
vendor/github.com/containers/common/libnetwork/internal/rootlessnetns/netns_linux.go
generated
vendored
|
@ -8,9 +8,9 @@ import (
|
|||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
"github.com/containers/common/libnetwork/pasta"
|
||||
"github.com/containers/common/libnetwork/resolvconf"
|
||||
"github.com/containers/common/libnetwork/slirp4netns"
|
||||
"github.com/containers/common/pkg/config"
|
||||
|
@ -31,8 +31,8 @@ const (
|
|||
// refCountFile file name for the ref count file
|
||||
refCountFile = "ref-count"
|
||||
|
||||
// rootlessNetNsSilrp4netnsPidFile is the name of the rootless netns slirp4netns pid file
|
||||
rootlessNetNsSilrp4netnsPidFile = "rootless-netns-slirp4netns.pid"
|
||||
// rootlessNetNsConnPidFile is the name of the rootless netns slirp4netns/pasta pid file
|
||||
rootlessNetNsConnPidFile = "rootless-netns-conn.pid"
|
||||
|
||||
// persistentCNIDir is the directory where the CNI files are stored
|
||||
persistentCNIDir = "/var/lib/cni"
|
||||
|
@ -113,7 +113,14 @@ func (n *Netns) getOrCreateNetns() (ns.NetNS, bool, error) {
|
|||
if err != nil {
|
||||
return nil, false, wrapError("create netns", err)
|
||||
}
|
||||
err = n.setupSlirp4netns(nsPath)
|
||||
switch strings.ToLower(n.config.Network.DefaultRootlessNetworkCmd) {
|
||||
case "", slirp4netns.BinaryName:
|
||||
err = n.setupSlirp4netns(nsPath)
|
||||
case pasta.BinaryName:
|
||||
err = n.setupPasta(nsPath)
|
||||
default:
|
||||
err = fmt.Errorf("invalid rootless network command %q", n.config.Network.DefaultRootlessNetworkCmd)
|
||||
}
|
||||
return netns, true, err
|
||||
}
|
||||
|
||||
|
@ -133,8 +140,8 @@ func (n *Netns) cleanup() error {
|
|||
if err := netns.UnmountNS(nsPath); err != nil {
|
||||
multiErr = multierror.Append(multiErr, err)
|
||||
}
|
||||
if err := n.cleanupSlirp4netns(); err != nil {
|
||||
multiErr = multierror.Append(multiErr, wrapError("kill slirp4netns", err))
|
||||
if err := n.cleanupRootlessNetns(); err != nil {
|
||||
multiErr = multierror.Append(multiErr, wrapError("kill network process", err))
|
||||
}
|
||||
if err := os.RemoveAll(n.dir); err != nil {
|
||||
multiErr = multierror.Append(multiErr, wrapError("remove rootless netns dir", err))
|
||||
|
@ -143,6 +150,53 @@ func (n *Netns) cleanup() error {
|
|||
return multiErr.ErrorOrNil()
|
||||
}
|
||||
|
||||
func (n *Netns) setupPasta(nsPath string) error {
|
||||
pidPath := n.getPath(rootlessNetNsConnPidFile)
|
||||
|
||||
pastaOpts := pasta.SetupOptions{
|
||||
Config: n.config,
|
||||
Netns: nsPath,
|
||||
ExtraOptions: []string{"--pid", pidPath},
|
||||
}
|
||||
if err := pasta.Setup(&pastaOpts); err != nil {
|
||||
return fmt.Errorf("setting up Pasta: %w", err)
|
||||
}
|
||||
|
||||
if systemd.RunsOnSystemd() {
|
||||
// Treat these as fatal - if pasta failed to write a PID file something is probably wrong.
|
||||
pidfile, err := os.ReadFile(pidPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open pasta PID file: %w", err)
|
||||
}
|
||||
pid, err := strconv.Atoi(strings.TrimSpace(string(pidfile)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to decode pasta PID: %w", err)
|
||||
}
|
||||
|
||||
if err := systemd.MoveRootlessNetnsSlirpProcessToUserSlice(pid); err != nil {
|
||||
// only log this, it is not fatal but can lead to issues when running podman inside systemd units
|
||||
logrus.Errorf("failed to move the rootless netns pasta process to the systemd user.slice: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := resolvconf.New(&resolvconf.Params{
|
||||
Path: n.getPath(resolvConfName),
|
||||
// fake the netns since we want to filter localhost
|
||||
Namespaces: []specs.LinuxNamespace{
|
||||
{Type: specs.NetworkNamespace},
|
||||
},
|
||||
// TODO: Need a way to determine if there is a valid v6 address on any
|
||||
// external interface of the system.
|
||||
IPv6Enabled: false,
|
||||
KeepHostServers: true,
|
||||
Nameservers: []string{},
|
||||
}); err != nil {
|
||||
return wrapError("create resolv.conf", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *Netns) setupSlirp4netns(nsPath string) error {
|
||||
res, err := slirp4netns.Setup(&slirp4netns.SetupOptions{
|
||||
Config: n.config,
|
||||
|
@ -155,7 +209,7 @@ func (n *Netns) setupSlirp4netns(nsPath string) error {
|
|||
// create pid file for the slirp4netns process
|
||||
// this is need to kill the process in the cleanup
|
||||
pid := strconv.Itoa(res.Pid)
|
||||
err = os.WriteFile(n.getPath(rootlessNetNsSilrp4netnsPidFile), []byte(pid), 0o600)
|
||||
err = os.WriteFile(n.getPath(rootlessNetNsConnPidFile), []byte(pid), 0o600)
|
||||
if err != nil {
|
||||
return wrapError("write slirp4netns pid file", err)
|
||||
}
|
||||
|
@ -190,15 +244,18 @@ func (n *Netns) setupSlirp4netns(nsPath string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (n *Netns) cleanupSlirp4netns() error {
|
||||
pidFile := n.getPath(rootlessNetNsSilrp4netnsPidFile)
|
||||
func (n *Netns) cleanupRootlessNetns() error {
|
||||
pidFile := n.getPath(rootlessNetNsConnPidFile)
|
||||
b, err := os.ReadFile(pidFile)
|
||||
if err == nil {
|
||||
var i int
|
||||
i, err = strconv.Atoi(string(b))
|
||||
i, err = strconv.Atoi(strings.TrimSpace(string(b)))
|
||||
if err == nil {
|
||||
// kill the slirp process so we do not leak it
|
||||
err = syscall.Kill(i, syscall.SIGTERM)
|
||||
err = unix.Kill(i, unix.SIGTERM)
|
||||
if err == unix.ESRCH {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return err
|
||||
|
|
|
@ -47,6 +47,7 @@ func Setup(opts *SetupOptions) error {
|
|||
NoTCPNamespacePorts := true
|
||||
NoUDPNamespacePorts := true
|
||||
NoMapGW := true
|
||||
NoDNS := true
|
||||
|
||||
path, err := opts.Config.FindHelperBinary(BinaryName, true)
|
||||
if err != nil {
|
||||
|
@ -102,6 +103,8 @@ func Setup(opts *SetupOptions) error {
|
|||
NoMapGW = false
|
||||
// not an actual pasta(1) option
|
||||
cmdArgs = append(cmdArgs[:i], cmdArgs[i+1:]...)
|
||||
case "-D", "--dns", "--dns-forward":
|
||||
NoDNS = false
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -120,21 +123,36 @@ func Setup(opts *SetupOptions) error {
|
|||
if NoMapGW {
|
||||
cmdArgs = append(cmdArgs, "--no-map-gw")
|
||||
}
|
||||
if NoDNS {
|
||||
// disable pasta reading from /etc/resolv.conf which hides the
|
||||
// "Couldn't get any nameserver address" warning when only
|
||||
// localhost resolvers are configured.
|
||||
cmdArgs = append(cmdArgs, "--dns", "none")
|
||||
}
|
||||
|
||||
cmdArgs = append(cmdArgs, "--netns", opts.Netns)
|
||||
// always pass --quiet to silence the info output from pasta
|
||||
cmdArgs = append(cmdArgs, "--quiet", "--netns", opts.Netns)
|
||||
|
||||
logrus.Debugf("pasta arguments: %s", strings.Join(cmdArgs, " "))
|
||||
|
||||
// pasta forks once ready, and quits once we delete the target namespace
|
||||
_, err = exec.Command(path, cmdArgs...).Output()
|
||||
out, err := exec.Command(path, cmdArgs...).CombinedOutput()
|
||||
if err != nil {
|
||||
exitErr := &exec.ExitError{}
|
||||
if errors.As(err, &exitErr) {
|
||||
return fmt.Errorf("pasta failed with exit code %d:\n%s",
|
||||
exitErr.ExitCode(), exitErr.Stderr)
|
||||
exitErr.ExitCode(), string(out))
|
||||
}
|
||||
return fmt.Errorf("failed to start pasta: %w", err)
|
||||
}
|
||||
|
||||
if len(out) > 0 {
|
||||
// TODO: This should be warning but right now pasta still prints
|
||||
// things with --quiet that we do not care about.
|
||||
// For now info is fine and we can bump it up later, it is only a
|
||||
// nice to have.
|
||||
logrus.Infof("pasta logged warnings: %q", string(out))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -13,6 +13,9 @@ import (
|
|||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/containers/storage/pkg/unshare"
|
||||
systemdDbus "github.com/coreos/go-systemd/v22/dbus"
|
||||
|
@ -22,6 +25,7 @@ import (
|
|||
"github.com/opencontainers/runc/libcontainer/configs"
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -30,6 +34,10 @@ var (
|
|||
// ErrCgroupV1Rootless means the cgroup v1 were attempted to be used in rootless environment
|
||||
ErrCgroupV1Rootless = errors.New("no support for CGroups V1 in rootless environments")
|
||||
ErrStatCgroup = errors.New("no cgroup available for gathering user statistics")
|
||||
|
||||
isUnifiedOnce sync.Once
|
||||
isUnified bool
|
||||
isUnifiedErr error
|
||||
)
|
||||
|
||||
// CgroupControl controls a cgroup hierarchy
|
||||
|
@ -731,3 +739,139 @@ func SystemCPUUsage() (uint64, error) {
|
|||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode.
|
||||
func IsCgroup2UnifiedMode() (bool, error) {
|
||||
isUnifiedOnce.Do(func() {
|
||||
var st syscall.Statfs_t
|
||||
if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil {
|
||||
isUnified, isUnifiedErr = false, err
|
||||
} else {
|
||||
isUnified, isUnifiedErr = st.Type == unix.CGROUP2_SUPER_MAGIC, nil
|
||||
}
|
||||
})
|
||||
return isUnified, isUnifiedErr
|
||||
}
|
||||
|
||||
// UserConnection returns an user connection to D-BUS
|
||||
func UserConnection(uid int) (*systemdDbus.Conn, error) {
|
||||
return systemdDbus.NewConnection(func() (*dbus.Conn, error) {
|
||||
return dbusAuthConnection(uid, dbus.SessionBusPrivateNoAutoStartup)
|
||||
})
|
||||
}
|
||||
|
||||
// UserOwnsCurrentSystemdCgroup checks whether the current EUID owns the
|
||||
// current cgroup.
|
||||
func UserOwnsCurrentSystemdCgroup() (bool, error) {
|
||||
uid := os.Geteuid()
|
||||
|
||||
cgroup2, err := IsCgroup2UnifiedMode()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
f, err := os.Open("/proc/self/cgroup")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
parts := strings.SplitN(line, ":", 3)
|
||||
|
||||
if len(parts) < 3 {
|
||||
continue
|
||||
}
|
||||
|
||||
var cgroupPath string
|
||||
|
||||
if cgroup2 {
|
||||
cgroupPath = filepath.Join(cgroupRoot, parts[2])
|
||||
} else {
|
||||
if parts[1] != "name=systemd" {
|
||||
continue
|
||||
}
|
||||
cgroupPath = filepath.Join(cgroupRoot, "systemd", parts[2])
|
||||
}
|
||||
|
||||
st, err := os.Stat(cgroupPath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
s := st.Sys()
|
||||
if s == nil {
|
||||
return false, fmt.Errorf("stat cgroup path %s", cgroupPath)
|
||||
}
|
||||
|
||||
if int(s.(*syscall.Stat_t).Uid) != uid {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return false, fmt.Errorf("parsing file /proc/self/cgroup: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// rmDirRecursively delete recursively a cgroup directory.
|
||||
// It differs from os.RemoveAll as it doesn't attempt to unlink files.
|
||||
// On cgroupfs we are allowed only to rmdir empty directories.
|
||||
func rmDirRecursively(path string) error {
|
||||
killProcesses := func(signal syscall.Signal) {
|
||||
if signal == unix.SIGKILL {
|
||||
if err := os.WriteFile(filepath.Join(path, "cgroup.kill"), []byte("1"), 0o600); err == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
// kill all the processes that are still part of the cgroup
|
||||
if procs, err := os.ReadFile(filepath.Join(path, "cgroup.procs")); err == nil {
|
||||
for _, pidS := range strings.Split(string(procs), "\n") {
|
||||
if pid, err := strconv.Atoi(pidS); err == nil {
|
||||
_ = unix.Kill(pid, signal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Remove(path); err == nil || errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
entries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, i := range entries {
|
||||
if i.IsDir() {
|
||||
if err := rmDirRecursively(filepath.Join(path, i.Name())); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
attempts := 0
|
||||
for {
|
||||
err := os.Remove(path)
|
||||
if err == nil || errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, unix.EBUSY) {
|
||||
// send a SIGTERM after 3 second
|
||||
if attempts == 300 {
|
||||
killProcesses(unix.SIGTERM)
|
||||
}
|
||||
// send SIGKILL after 8 seconds
|
||||
if attempts == 800 {
|
||||
killProcesses(unix.SIGKILL)
|
||||
}
|
||||
// give up after 10 seconds
|
||||
if attempts < 1000 {
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
attempts++
|
||||
continue
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("remove %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,162 +0,0 @@
|
|||
//go:build linux
|
||||
|
||||
package cgroups
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
systemdDbus "github.com/coreos/go-systemd/v22/dbus"
|
||||
"github.com/godbus/dbus/v5"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
var (
|
||||
isUnifiedOnce sync.Once
|
||||
isUnified bool
|
||||
isUnifiedErr error
|
||||
)
|
||||
|
||||
// IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode.
|
||||
func IsCgroup2UnifiedMode() (bool, error) {
|
||||
isUnifiedOnce.Do(func() {
|
||||
var st syscall.Statfs_t
|
||||
if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil {
|
||||
isUnified, isUnifiedErr = false, err
|
||||
} else {
|
||||
isUnified, isUnifiedErr = st.Type == unix.CGROUP2_SUPER_MAGIC, nil
|
||||
}
|
||||
})
|
||||
return isUnified, isUnifiedErr
|
||||
}
|
||||
|
||||
// UserConnection returns an user connection to D-BUS
|
||||
func UserConnection(uid int) (*systemdDbus.Conn, error) {
|
||||
return systemdDbus.NewConnection(func() (*dbus.Conn, error) {
|
||||
return dbusAuthConnection(uid, dbus.SessionBusPrivateNoAutoStartup)
|
||||
})
|
||||
}
|
||||
|
||||
// UserOwnsCurrentSystemdCgroup checks whether the current EUID owns the
|
||||
// current cgroup.
|
||||
func UserOwnsCurrentSystemdCgroup() (bool, error) {
|
||||
uid := os.Geteuid()
|
||||
|
||||
cgroup2, err := IsCgroup2UnifiedMode()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
f, err := os.Open("/proc/self/cgroup")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
parts := strings.SplitN(line, ":", 3)
|
||||
|
||||
if len(parts) < 3 {
|
||||
continue
|
||||
}
|
||||
|
||||
var cgroupPath string
|
||||
|
||||
if cgroup2 {
|
||||
cgroupPath = filepath.Join(cgroupRoot, parts[2])
|
||||
} else {
|
||||
if parts[1] != "name=systemd" {
|
||||
continue
|
||||
}
|
||||
cgroupPath = filepath.Join(cgroupRoot, "systemd", parts[2])
|
||||
}
|
||||
|
||||
st, err := os.Stat(cgroupPath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
s := st.Sys()
|
||||
if s == nil {
|
||||
return false, fmt.Errorf("stat cgroup path %s", cgroupPath)
|
||||
}
|
||||
|
||||
if int(s.(*syscall.Stat_t).Uid) != uid {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return false, fmt.Errorf("parsing file /proc/self/cgroup: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// rmDirRecursively delete recursively a cgroup directory.
|
||||
// It differs from os.RemoveAll as it doesn't attempt to unlink files.
|
||||
// On cgroupfs we are allowed only to rmdir empty directories.
|
||||
func rmDirRecursively(path string) error {
|
||||
killProcesses := func(signal syscall.Signal) {
|
||||
if signal == unix.SIGKILL {
|
||||
if err := os.WriteFile(filepath.Join(path, "cgroup.kill"), []byte("1"), 0o600); err == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
// kill all the processes that are still part of the cgroup
|
||||
if procs, err := os.ReadFile(filepath.Join(path, "cgroup.procs")); err == nil {
|
||||
for _, pidS := range strings.Split(string(procs), "\n") {
|
||||
if pid, err := strconv.Atoi(pidS); err == nil {
|
||||
_ = unix.Kill(pid, signal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Remove(path); err == nil || errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
entries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, i := range entries {
|
||||
if i.IsDir() {
|
||||
if err := rmDirRecursively(filepath.Join(path, i.Name())); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
attempts := 0
|
||||
for {
|
||||
err := os.Remove(path)
|
||||
if err == nil || errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, unix.EBUSY) {
|
||||
// send a SIGTERM after 3 second
|
||||
if attempts == 300 {
|
||||
killProcesses(unix.SIGTERM)
|
||||
}
|
||||
// send SIGKILL after 8 seconds
|
||||
if attempts == 800 {
|
||||
killProcesses(unix.SIGKILL)
|
||||
}
|
||||
// give up after 10 seconds
|
||||
if attempts < 1000 {
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
attempts++
|
||||
continue
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("remove %s: %w", path, err)
|
||||
}
|
||||
}
|
|
@ -3,10 +3,7 @@
|
|||
package cgroups
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
systemdDbus "github.com/coreos/go-systemd/v22/dbus"
|
||||
)
|
||||
|
||||
// IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode.
|
||||
|
@ -23,8 +20,3 @@ func UserOwnsCurrentSystemdCgroup() (bool, error) {
|
|||
func rmDirRecursively(path string) error {
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
|
||||
// UserConnection returns an user connection to D-BUS
|
||||
func UserConnection(uid int) (*systemdDbus.Conn, error) {
|
||||
return nil, fmt.Errorf("systemd d-bus is not supported on this platform")
|
||||
}
|
||||
|
|
|
@ -1,80 +0,0 @@
|
|||
//go:build !linux
|
||||
|
||||
package cgroups
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
systemdDbus "github.com/coreos/go-systemd/v22/dbus"
|
||||
"github.com/godbus/dbus/v5"
|
||||
)
|
||||
|
||||
func systemdCreate(path string, c *systemdDbus.Conn) error {
|
||||
slice, name := filepath.Split(path)
|
||||
slice = strings.TrimSuffix(slice, "/")
|
||||
|
||||
var lastError error
|
||||
for i := 0; i < 2; i++ {
|
||||
properties := []systemdDbus.Property{
|
||||
systemdDbus.PropDescription(fmt.Sprintf("cgroup %s", name)),
|
||||
systemdDbus.PropWants(slice),
|
||||
}
|
||||
pMap := map[string]bool{
|
||||
"DefaultDependencies": false,
|
||||
"MemoryAccounting": true,
|
||||
"CPUAccounting": true,
|
||||
"BlockIOAccounting": true,
|
||||
}
|
||||
if i == 0 {
|
||||
pMap["Delegate"] = true
|
||||
}
|
||||
for k, v := range pMap {
|
||||
p := systemdDbus.Property{
|
||||
Name: k,
|
||||
Value: dbus.MakeVariant(v),
|
||||
}
|
||||
properties = append(properties, p)
|
||||
}
|
||||
|
||||
ch := make(chan string)
|
||||
_, err := c.StartTransientUnitContext(context.TODO(), name, "replace", properties, ch)
|
||||
if err != nil {
|
||||
lastError = err
|
||||
continue
|
||||
}
|
||||
<-ch
|
||||
return nil
|
||||
}
|
||||
return lastError
|
||||
}
|
||||
|
||||
/*
|
||||
systemdDestroyConn is copied from containerd/cgroups/systemd.go file, that
|
||||
has the following license:
|
||||
Copyright The containerd Authors.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
func systemdDestroyConn(path string, c *systemdDbus.Conn) error {
|
||||
name := filepath.Base(path)
|
||||
|
||||
ch := make(chan string)
|
||||
_, err := c.StopUnitContext(context.TODO(), name, "replace", ch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
<-ch
|
||||
return nil
|
||||
}
|
|
@ -370,11 +370,6 @@ type EngineConfig struct {
|
|||
// LockType is the type of locking to use.
|
||||
LockType string `toml:"lock_type,omitempty"`
|
||||
|
||||
// MachineEnabled indicates if Podman is running in a podman-machine VM
|
||||
//
|
||||
// This method is soft deprecated, use machine.IsPodmanMachine instead
|
||||
MachineEnabled bool `toml:"machine_enabled,omitempty"`
|
||||
|
||||
// MultiImageArchive - if true, the container engine allows for storing
|
||||
// archives (e.g., of the docker-archive transport) with multiple
|
||||
// images. By default, Podman creates single-image archives.
|
||||
|
@ -668,6 +663,8 @@ type MachineConfig struct {
|
|||
Volumes attributedstring.Slice `toml:"volumes,omitempty"`
|
||||
// Provider is the virtualization provider used to run podman-machine VM
|
||||
Provider string `toml:"provider,omitempty"`
|
||||
// Rosetta is the flag to enable Rosetta in the podman-machine VM on Apple Silicon
|
||||
Rosetta bool `toml:"rosetta,omitempty"`
|
||||
}
|
||||
|
||||
// FarmConfig represents the "farm" TOML config tables
|
||||
|
|
|
@ -384,9 +384,9 @@ default_sysctls = [
|
|||
|
||||
|
||||
# Configure which rootless network program to use by default. Valid options are
|
||||
# `slirp4netns` (default) and `pasta`.
|
||||
# `slirp4netns` and `pasta` (default).
|
||||
#
|
||||
#default_rootless_network_cmd = "slirp4netns"
|
||||
#default_rootless_network_cmd = "pasta"
|
||||
|
||||
# Path to the directory where network configuration files are located.
|
||||
# For the CNI backend the default is "/etc/cni/net.d" as root
|
||||
|
@ -435,6 +435,9 @@ default_sysctls = [
|
|||
|
||||
# The compression format to use when pushing an image.
|
||||
# Valid options are: `gzip`, `zstd` and `zstd:chunked`.
|
||||
# This field is ignored when pushing images to the docker-daemon and
|
||||
# docker-archive formats. It is also ignored when the manifest format is set
|
||||
# to v2s2.
|
||||
#
|
||||
#compression_format = "gzip"
|
||||
|
||||
|
@ -662,7 +665,7 @@ default_sysctls = [
|
|||
|
||||
# Delay between retries in case pulling/pushing image fails.
|
||||
# If set, container engines will retry at the set interval,
|
||||
# otherwise they delay 2 seconds and then exponentially back off.
|
||||
# otherwise they delay 2 seconds and then exponentially back off.
|
||||
#
|
||||
#retry_delay = "2s"
|
||||
|
||||
|
@ -820,16 +823,15 @@ default_sysctls = [
|
|||
#
|
||||
#disk_size=10
|
||||
|
||||
# Default image URI when creating a new VM using `podman machine init`.
|
||||
# Options: On Linux/Mac, `testing`, `stable`, `next`. On Windows, the major
|
||||
# version of the OS (e.g `36`) for Fedora 36. For all platforms you can
|
||||
# alternatively specify a custom download URL to an image. Container engines
|
||||
# translate URIs $OS and $ARCH to the native OS and ARCH. URI
|
||||
# "https://example.com/$OS/$ARCH/foobar.ami" becomes
|
||||
# Default Image used when creating a new VM using `podman machine init`.
|
||||
# Can be specified as registry with a bootable OCI artifact, download URL, or a local path.
|
||||
# Registry target must be in the form of `docker://registry/repo/image:version`.
|
||||
# Container engines translate URIs $OS and $ARCH to the native OS and ARCH.
|
||||
# URI "https://example.com/$OS/$ARCH/foobar.ami" would become
|
||||
# "https://example.com/linux/amd64/foobar.ami" on a Linux AMD machine.
|
||||
# The default value is `testing`.
|
||||
# If unspecified, the default Podman machine image will be used.
|
||||
#
|
||||
#image = "testing"
|
||||
#image = ""
|
||||
|
||||
# Memory in MB a machine is created with.
|
||||
#
|
||||
|
@ -854,6 +856,11 @@ default_sysctls = [
|
|||
#
|
||||
#provider = ""
|
||||
|
||||
# Rosetta supports running x86_64 Linux binaries on a Podman machine on Apple silicon.
|
||||
# The default value is `true`. Supported on AppleHV(arm64) machines only.
|
||||
#
|
||||
#rosetta=true
|
||||
|
||||
# The [machine] table MUST be the last entry in this file.
|
||||
# (Unless another table is added)
|
||||
# TOML does not provide a way to end a table other than a further table being
|
||||
|
|
|
@ -257,7 +257,7 @@ func defaultConfig() (*Config, error) {
|
|||
DefaultNetwork: "podman",
|
||||
DefaultSubnet: DefaultSubnet,
|
||||
DefaultSubnetPools: DefaultSubnetPools,
|
||||
DefaultRootlessNetworkCmd: "slirp4netns",
|
||||
DefaultRootlessNetworkCmd: "pasta",
|
||||
DNSBindPort: 0,
|
||||
CNIPluginDirs: attributedstring.NewSlice(DefaultCNIPluginDirs),
|
||||
NetavarkPluginDirs: attributedstring.NewSlice(DefaultNetavarkPluginDirs),
|
||||
|
@ -286,10 +286,14 @@ func defaultMachineConfig() MachineConfig {
|
|||
return MachineConfig{
|
||||
CPUs: uint64(cpus),
|
||||
DiskSize: 100,
|
||||
Image: getDefaultMachineImage(),
|
||||
Memory: 2048,
|
||||
User: getDefaultMachineUser(),
|
||||
Volumes: attributedstring.NewSlice(getDefaultMachineVolumes()),
|
||||
// TODO: Set machine image default here
|
||||
// Currently the default is set in Podman as we need time to stabilize
|
||||
// VM images and locations between different providers.
|
||||
Image: "",
|
||||
Memory: 2048,
|
||||
User: getDefaultMachineUser(),
|
||||
Volumes: attributedstring.NewSlice(getDefaultMachineVolumes()),
|
||||
Rosetta: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -480,7 +484,6 @@ func defaultEngineConfig() (*EngineConfig, error) {
|
|||
// TODO - ideally we should expose a `type LockType string` along with
|
||||
// constants.
|
||||
c.LockType = getDefaultLockType()
|
||||
c.MachineEnabled = false
|
||||
c.ChownCopiedFiles = true
|
||||
|
||||
c.PodExitPolicy = defaultPodExitPolicy
|
||||
|
@ -649,11 +652,6 @@ func (c *Config) LogDriver() string {
|
|||
return c.Containers.LogDriver
|
||||
}
|
||||
|
||||
// MachineEnabled returns if podman is running inside a VM or not.
|
||||
func (c *Config) MachineEnabled() bool {
|
||||
return c.Engine.MachineEnabled
|
||||
}
|
||||
|
||||
// MachineVolumes returns volumes to mount into the VM.
|
||||
func (c *Config) MachineVolumes() ([]string, error) {
|
||||
return machineVolumes(c.Machine.Volumes.Get())
|
||||
|
@ -683,12 +681,6 @@ func getDefaultSSHConfig() string {
|
|||
return filepath.Join(dirname, ".ssh", "config")
|
||||
}
|
||||
|
||||
// getDefaultImage returns the default machine image stream
|
||||
// On Windows this refers to the Fedora major release number
|
||||
func getDefaultMachineImage() string {
|
||||
return "testing"
|
||||
}
|
||||
|
||||
// getDefaultMachineUser returns the user to use for rootless podman
|
||||
// This is only for the apple, hyperv, and qemu implementations.
|
||||
// WSL's user will be hardcoded in podman to "user"
|
||||
|
|
|
@ -4,9 +4,6 @@ import (
|
|||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/containers/common/pkg/config"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Marker struct {
|
||||
|
@ -29,9 +26,7 @@ var (
|
|||
|
||||
func loadMachineMarker(file string) {
|
||||
var kind string
|
||||
|
||||
// Support deprecated config value for compatibility
|
||||
enabled := isLegacyConfigSet()
|
||||
enabled := false
|
||||
|
||||
if content, err := os.ReadFile(file); err == nil {
|
||||
enabled = true
|
||||
|
@ -41,17 +36,6 @@ func loadMachineMarker(file string) {
|
|||
marker = &Marker{enabled, kind}
|
||||
}
|
||||
|
||||
func isLegacyConfigSet() bool {
|
||||
config, err := config.Default()
|
||||
if err != nil {
|
||||
logrus.Warnf("could not obtain container configuration")
|
||||
return false
|
||||
}
|
||||
|
||||
//nolint:staticcheck //lint:ignore SA1019 deprecated call
|
||||
return config.Engine.MachineEnabled
|
||||
}
|
||||
|
||||
func IsPodmanMachine() bool {
|
||||
return GetMachineMarker().Enabled
|
||||
}
|
||||
|
|
|
@ -177,16 +177,26 @@ func newNSPath(nsPath string) (ns.NetNS, error) {
|
|||
|
||||
// UnmountNS unmounts the given netns path
|
||||
func UnmountNS(nsPath string) error {
|
||||
var rErr error
|
||||
// Only unmount if it's been bind-mounted (don't touch namespaces in /proc...)
|
||||
if !strings.HasPrefix(nsPath, "/proc/") {
|
||||
if err := unix.Unmount(nsPath, unix.MNT_DETACH); err != nil {
|
||||
return fmt.Errorf("failed to unmount NS: at %s: %v", nsPath, err)
|
||||
// Do not return here, always try to remove below.
|
||||
// This is important in case podman now is in a new userns compared to
|
||||
// when the netns was created. The umount will fail EINVAL but removing
|
||||
// the file will work and the kernel will destroy the bind mount in the
|
||||
// other ns because of this. We also need it so pasta doesn't leak.
|
||||
rErr = fmt.Errorf("failed to unmount NS: at %s: %w", nsPath, err)
|
||||
}
|
||||
|
||||
if err := os.Remove(nsPath); err != nil {
|
||||
return fmt.Errorf("failed to remove ns path %s: %v", nsPath, err)
|
||||
err := fmt.Errorf("failed to remove ns path: %w", err)
|
||||
if rErr != nil {
|
||||
err = fmt.Errorf("%v, %w", err, rErr)
|
||||
}
|
||||
rErr = err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return rErr
|
||||
}
|
||||
|
|
|
@ -106,7 +106,7 @@ github.com/containernetworking/cni/pkg/version
|
|||
# github.com/containernetworking/plugins v1.4.0
|
||||
## explicit; go 1.20
|
||||
github.com/containernetworking/plugins/pkg/ns
|
||||
# github.com/containers/common v0.57.1-0.20240220203037-6ee157e78afb
|
||||
# github.com/containers/common v0.57.1-0.20240301113114-0b996b05cd16
|
||||
## explicit; go 1.20
|
||||
github.com/containers/common/internal
|
||||
github.com/containers/common/internal/attributedstring
|
||||
|
|
Loading…
Reference in New Issue