| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | /* | 
					
						
							|  |  |  | Copyright 2016 The Kubernetes Authors. | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | Licensed under the Apache License, Version 2.0 (the "License"); | 
					
						
							|  |  |  | you may not use this file except in compliance with the License. | 
					
						
							|  |  |  | You may obtain a copy of the License at | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     http://www.apache.org/licenses/LICENSE-2.0
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | Unless required by applicable law or agreed to in writing, software | 
					
						
							|  |  |  | distributed under the License is distributed on an "AS IS" BASIS, | 
					
						
							|  |  |  | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 
					
						
							|  |  |  | See the License for the specific language governing permissions and | 
					
						
							|  |  |  | limitations under the License. | 
					
						
							|  |  |  | */ | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | package kubelet | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | import ( | 
					
						
							| 
									
										
										
										
											2016-07-27 16:53:18 +08:00
										 |  |  | 	"fmt" | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 	"net" | 
					
						
							|  |  |  | 	"strings" | 
					
						
							|  |  |  | 	"testing" | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-03-07 04:30:31 +08:00
										 |  |  | 	"github.com/stretchr/testify/assert" | 
					
						
							|  |  |  | 	"github.com/stretchr/testify/require" | 
					
						
							| 
									
										
										
										
											2017-01-31 02:39:54 +08:00
										 |  |  | 	"k8s.io/client-go/tools/record" | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 	"k8s.io/kubernetes/pkg/api/v1" | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 	"k8s.io/kubernetes/pkg/util/bandwidth" | 
					
						
							|  |  |  | ) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | func TestNodeIPParam(t *testing.T) { | 
					
						
							|  |  |  | 	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) | 
					
						
							| 
									
										
										
										
											2016-12-15 06:31:01 +08:00
										 |  |  | 	defer testKubelet.Cleanup() | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 	kubelet := testKubelet.kubelet | 
					
						
							|  |  |  | 	tests := []struct { | 
					
						
							|  |  |  | 		nodeIP   string | 
					
						
							|  |  |  | 		success  bool | 
					
						
							|  |  |  | 		testName string | 
					
						
							|  |  |  | 	}{ | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			nodeIP:   "", | 
					
						
							|  |  |  | 			success:  true, | 
					
						
							|  |  |  | 			testName: "IP not set", | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			nodeIP:   "127.0.0.1", | 
					
						
							|  |  |  | 			success:  false, | 
					
						
							|  |  |  | 			testName: "loopback address", | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			nodeIP:   "FE80::0202:B3FF:FE1E:8329", | 
					
						
							|  |  |  | 			success:  false, | 
					
						
							|  |  |  | 			testName: "IPv6 address", | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			nodeIP:   "1.2.3.4", | 
					
						
							|  |  |  | 			success:  false, | 
					
						
							|  |  |  | 			testName: "IPv4 address that doesn't belong to host", | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 	for _, test := range tests { | 
					
						
							|  |  |  | 		kubelet.nodeIP = net.ParseIP(test.nodeIP) | 
					
						
							|  |  |  | 		err := kubelet.validateNodeIP() | 
					
						
							| 
									
										
										
										
											2017-03-07 04:30:31 +08:00
										 |  |  | 		if test.success { | 
					
						
							|  |  |  | 			assert.NoError(t, err, "test %s", test.testName) | 
					
						
							|  |  |  | 		} else { | 
					
						
							|  |  |  | 			assert.Error(t, err, fmt.Sprintf("test %s", test.testName)) | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 		} | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | func TestParseResolvConf(t *testing.T) { | 
					
						
							|  |  |  | 	testCases := []struct { | 
					
						
							|  |  |  | 		data        string | 
					
						
							|  |  |  | 		nameservers []string | 
					
						
							|  |  |  | 		searches    []string | 
					
						
							|  |  |  | 	}{ | 
					
						
							|  |  |  | 		{"", []string{}, []string{}}, | 
					
						
							|  |  |  | 		{" ", []string{}, []string{}}, | 
					
						
							|  |  |  | 		{"\n", []string{}, []string{}}, | 
					
						
							|  |  |  | 		{"\t\n\t", []string{}, []string{}}, | 
					
						
							|  |  |  | 		{"#comment\n", []string{}, []string{}}, | 
					
						
							|  |  |  | 		{" #comment\n", []string{}, []string{}}, | 
					
						
							|  |  |  | 		{"#comment\n#comment", []string{}, []string{}}, | 
					
						
							|  |  |  | 		{"#comment\nnameserver", []string{}, []string{}}, | 
					
						
							|  |  |  | 		{"#comment\nnameserver\nsearch", []string{}, []string{}}, | 
					
						
							|  |  |  | 		{"nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}}, | 
					
						
							|  |  |  | 		{" nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}}, | 
					
						
							|  |  |  | 		{"\tnameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}}, | 
					
						
							|  |  |  | 		{"nameserver\t1.2.3.4", []string{"1.2.3.4"}, []string{}}, | 
					
						
							|  |  |  | 		{"nameserver \t 1.2.3.4", []string{"1.2.3.4"}, []string{}}, | 
					
						
							|  |  |  | 		{"nameserver 1.2.3.4\nnameserver 5.6.7.8", []string{"1.2.3.4", "5.6.7.8"}, []string{}}, | 
					
						
							|  |  |  | 		{"search foo", []string{}, []string{"foo"}}, | 
					
						
							|  |  |  | 		{"search foo bar", []string{}, []string{"foo", "bar"}}, | 
					
						
							|  |  |  | 		{"search foo bar bat\n", []string{}, []string{"foo", "bar", "bat"}}, | 
					
						
							|  |  |  | 		{"search foo\nsearch bar", []string{}, []string{"bar"}}, | 
					
						
							|  |  |  | 		{"nameserver 1.2.3.4\nsearch foo bar", []string{"1.2.3.4"}, []string{"foo", "bar"}}, | 
					
						
							|  |  |  | 		{"nameserver 1.2.3.4\nsearch foo\nnameserver 5.6.7.8\nsearch bar", []string{"1.2.3.4", "5.6.7.8"}, []string{"bar"}}, | 
					
						
							|  |  |  | 		{"#comment\nnameserver 1.2.3.4\n#comment\nsearch foo\ncomment", []string{"1.2.3.4"}, []string{"foo"}}, | 
					
						
							|  |  |  | 	} | 
					
						
							| 
									
										
										
										
											2016-11-09 08:19:53 +08:00
										 |  |  | 	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) | 
					
						
							| 
									
										
										
										
											2016-12-15 06:31:01 +08:00
										 |  |  | 	defer testKubelet.Cleanup() | 
					
						
							| 
									
										
										
										
											2016-11-09 08:19:53 +08:00
										 |  |  | 	kubelet := testKubelet.kubelet | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 	for i, tc := range testCases { | 
					
						
							| 
									
										
										
										
											2016-11-09 08:19:53 +08:00
										 |  |  | 		ns, srch, err := kubelet.parseResolvConf(strings.NewReader(tc.data)) | 
					
						
							| 
									
										
										
										
											2017-03-07 04:30:31 +08:00
										 |  |  | 		require.NoError(t, err) | 
					
						
							|  |  |  | 		assert.EqualValues(t, tc.nameservers, ns, "test case [%d]: name servers", i) | 
					
						
							|  |  |  | 		assert.EqualValues(t, tc.searches, srch, "test case [%d] searches", i) | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-07-27 16:53:18 +08:00
										 |  |  | func TestComposeDNSSearch(t *testing.T) { | 
					
						
							|  |  |  | 	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) | 
					
						
							|  |  |  | 	kubelet := testKubelet.kubelet | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	recorder := record.NewFakeRecorder(20) | 
					
						
							|  |  |  | 	kubelet.recorder = recorder | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	pod := podWithUidNameNs("", "test_pod", "testNS") | 
					
						
							|  |  |  | 	kubelet.clusterDomain = "TEST" | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	testCases := []struct { | 
					
						
							|  |  |  | 		dnsNames     []string | 
					
						
							|  |  |  | 		hostNames    []string | 
					
						
							|  |  |  | 		resultSearch []string | 
					
						
							|  |  |  | 		events       []string | 
					
						
							|  |  |  | 	}{ | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"}, | 
					
						
							|  |  |  | 			[]string{}, | 
					
						
							|  |  |  | 			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"}, | 
					
						
							|  |  |  | 			[]string{}, | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"}, | 
					
						
							|  |  |  | 			[]string{"AAA", "svc.TEST", "BBB", "TEST"}, | 
					
						
							|  |  |  | 			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA", "BBB"}, | 
					
						
							|  |  |  | 			[]string{ | 
					
						
							|  |  |  | 				"Found and omitted duplicated dns domain in host search line: 'svc.TEST' during merging with cluster dns domains", | 
					
						
							|  |  |  | 				"Found and omitted duplicated dns domain in host search line: 'TEST' during merging with cluster dns domains", | 
					
						
							|  |  |  | 			}, | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"}, | 
					
						
							|  |  |  | 			[]string{"AAA", strings.Repeat("B", 256), "BBB"}, | 
					
						
							|  |  |  | 			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA"}, | 
					
						
							|  |  |  | 			[]string{"Search Line limits were exceeded, some dns names have been omitted, the applied search line is: testNS.svc.TEST svc.TEST TEST AAA"}, | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"}, | 
					
						
							|  |  |  | 			[]string{"AAA", "TEST", "BBB", "TEST", "CCC", "DDD"}, | 
					
						
							|  |  |  | 			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA", "BBB", "CCC"}, | 
					
						
							|  |  |  | 			[]string{ | 
					
						
							|  |  |  | 				"Found and omitted duplicated dns domain in host search line: 'TEST' during merging with cluster dns domains", | 
					
						
							|  |  |  | 				"Found and omitted duplicated dns domain in host search line: 'TEST' during merging with cluster dns domains", | 
					
						
							|  |  |  | 				"Search Line limits were exceeded, some dns names have been omitted, the applied search line is: testNS.svc.TEST svc.TEST TEST AAA BBB CCC", | 
					
						
							|  |  |  | 			}, | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	fetchEvent := func(recorder *record.FakeRecorder) string { | 
					
						
							|  |  |  | 		select { | 
					
						
							|  |  |  | 		case event := <-recorder.Events: | 
					
						
							|  |  |  | 			return event | 
					
						
							|  |  |  | 		default: | 
					
						
							|  |  |  | 			return "No more events!" | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	for i, tc := range testCases { | 
					
						
							|  |  |  | 		dnsSearch := kubelet.formDNSSearch(tc.hostNames, pod) | 
					
						
							| 
									
										
										
										
											2017-03-07 04:30:31 +08:00
										 |  |  | 		assert.EqualValues(t, tc.resultSearch, dnsSearch, "test [%d]", i) | 
					
						
							| 
									
										
										
										
											2016-07-27 16:53:18 +08:00
										 |  |  | 		for _, expectedEvent := range tc.events { | 
					
						
							|  |  |  | 			expected := fmt.Sprintf("%s %s %s", v1.EventTypeWarning, "DNSSearchForming", expectedEvent) | 
					
						
							|  |  |  | 			event := fetchEvent(recorder) | 
					
						
							| 
									
										
										
										
											2017-03-07 04:30:31 +08:00
										 |  |  | 			assert.Equal(t, expected, event, "test [%d]", i) | 
					
						
							| 
									
										
										
										
											2016-07-27 16:53:18 +08:00
										 |  |  | 		} | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | func TestCleanupBandwidthLimits(t *testing.T) { | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 	testPod := func(name, ingress string) *v1.Pod { | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 		pod := podWithUidNameNs("", name, "") | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		if len(ingress) != 0 { | 
					
						
							|  |  |  | 			pod.Annotations["kubernetes.io/ingress-bandwidth"] = ingress | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		return pod | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	// TODO(random-liu): We removed the test case for pod status not cached here. We should add a higher
 | 
					
						
							|  |  |  | 	// layer status getter function and test that function instead.
 | 
					
						
							|  |  |  | 	tests := []struct { | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 		status           *v1.PodStatus | 
					
						
							|  |  |  | 		pods             []*v1.Pod | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 		inputCIDRs       []string | 
					
						
							|  |  |  | 		expectResetCIDRs []string | 
					
						
							|  |  |  | 		name             string | 
					
						
							|  |  |  | 	}{ | 
					
						
							|  |  |  | 		{ | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 			status: &v1.PodStatus{ | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 				PodIP: "1.2.3.4", | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 				Phase: v1.PodRunning, | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 			}, | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 			pods: []*v1.Pod{ | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 				testPod("foo", "10M"), | 
					
						
							|  |  |  | 				testPod("bar", ""), | 
					
						
							|  |  |  | 			}, | 
					
						
							|  |  |  | 			inputCIDRs:       []string{"1.2.3.4/32", "2.3.4.5/32", "5.6.7.8/32"}, | 
					
						
							|  |  |  | 			expectResetCIDRs: []string{"2.3.4.5/32", "5.6.7.8/32"}, | 
					
						
							|  |  |  | 			name:             "pod running", | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 		{ | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 			status: &v1.PodStatus{ | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 				PodIP: "1.2.3.4", | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 				Phase: v1.PodFailed, | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 			}, | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 			pods: []*v1.Pod{ | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 				testPod("foo", "10M"), | 
					
						
							|  |  |  | 				testPod("bar", ""), | 
					
						
							|  |  |  | 			}, | 
					
						
							|  |  |  | 			inputCIDRs:       []string{"1.2.3.4/32", "2.3.4.5/32", "5.6.7.8/32"}, | 
					
						
							|  |  |  | 			expectResetCIDRs: []string{"1.2.3.4/32", "2.3.4.5/32", "5.6.7.8/32"}, | 
					
						
							|  |  |  | 			name:             "pod not running", | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 		{ | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 			status: &v1.PodStatus{ | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 				PodIP: "1.2.3.4", | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 				Phase: v1.PodFailed, | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 			}, | 
					
						
							| 
									
										
										
										
											2016-11-19 04:50:58 +08:00
										 |  |  | 			pods: []*v1.Pod{ | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 				testPod("foo", ""), | 
					
						
							|  |  |  | 				testPod("bar", ""), | 
					
						
							|  |  |  | 			}, | 
					
						
							|  |  |  | 			inputCIDRs:       []string{"1.2.3.4/32", "2.3.4.5/32", "5.6.7.8/32"}, | 
					
						
							|  |  |  | 			expectResetCIDRs: []string{"1.2.3.4/32", "2.3.4.5/32", "5.6.7.8/32"}, | 
					
						
							|  |  |  | 			name:             "no bandwidth limits", | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 	for _, test := range tests { | 
					
						
							|  |  |  | 		shaper := &bandwidth.FakeShaper{ | 
					
						
							|  |  |  | 			CIDRs: test.inputCIDRs, | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		testKube := newTestKubelet(t, false /* controllerAttachDetachEnabled */) | 
					
						
							| 
									
										
										
										
											2016-12-15 06:31:01 +08:00
										 |  |  | 		defer testKube.Cleanup() | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 		testKube.kubelet.shaper = shaper | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		for _, pod := range test.pods { | 
					
						
							|  |  |  | 			testKube.kubelet.statusManager.SetPodStatus(pod, *test.status) | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		err := testKube.kubelet.cleanupBandwidthLimits(test.pods) | 
					
						
							| 
									
										
										
										
											2017-03-07 04:30:31 +08:00
										 |  |  | 		assert.NoError(t, err, "test [%s]", test.name) | 
					
						
							|  |  |  | 		assert.EqualValues(t, test.expectResetCIDRs, shaper.ResetCIDRs, "test[%s]", test.name) | 
					
						
							| 
									
										
										
										
											2016-07-06 05:43:42 +08:00
										 |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							| 
									
										
										
										
											2016-08-18 04:01:27 +08:00
										 |  |  | 
 | 
					
						
							|  |  |  | func TestGetIPTablesMark(t *testing.T) { | 
					
						
							|  |  |  | 	tests := []struct { | 
					
						
							|  |  |  | 		bit    int | 
					
						
							|  |  |  | 		expect string | 
					
						
							|  |  |  | 	}{ | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			14, | 
					
						
							|  |  |  | 			"0x00004000/0x00004000", | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 		{ | 
					
						
							|  |  |  | 			15, | 
					
						
							|  |  |  | 			"0x00008000/0x00008000", | 
					
						
							|  |  |  | 		}, | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 	for _, tc := range tests { | 
					
						
							|  |  |  | 		res := getIPTablesMark(tc.bit) | 
					
						
							| 
									
										
										
										
											2017-03-07 04:30:31 +08:00
										 |  |  | 		assert.Equal(t, tc.expect, res, "input %d", tc.bit) | 
					
						
							| 
									
										
										
										
											2016-08-18 04:01:27 +08:00
										 |  |  | 	} | 
					
						
							|  |  |  | } |