github-vet / preliminary-findings-jan-2021 Goto Github PK

for pKeyStr, pEntry := range mp {
	var pKey peerKey
	if _, err := fmt.Sscan(pKeyStr, &pKey); err != nil {
		logrus.Warnf("Peer key scan on network %s failed: %v", nid, err)
	}
	if f(&pKey, &pEntry) {
		return nil
	}
}

for _, log := range logs {
	notifier.Notify(rpcSub.ID, &log)
}

for i, dig := range digests {
	res[i] = &dig
}

for _, v := range s {
	go func() {
		println(v) // ERROR "range variable v captured by func literal"
	}()
}

for _, mountCandidate := range candidates {
	logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountCandidate.Digest, mountCandidate.SourceRepository)
	createOpts := []distribution.BlobCreateOption{}

	if len(mountCandidate.SourceRepository) > 0 {
		namedRef, err := reference.ParseNormalizedNamed(mountCandidate.SourceRepository)
		if err != nil {
			logrus.Errorf("failed to parse source repository reference %v: %v", reference.FamiliarString(namedRef), err)
			pd.v2MetadataService.Remove(mountCandidate)
			continue
		}

		// Candidates are always under same domain, create remote reference
		// with only path to set mount from with
		remoteRef, err := reference.WithName(reference.Path(namedRef))
		if err != nil {
			logrus.Errorf("failed to make remote reference out of %q: %v", reference.Path(namedRef), err)
			continue
		}

		canonicalRef, err := reference.WithDigest(reference.TrimNamed(remoteRef), mountCandidate.Digest)
		if err != nil {
			logrus.Errorf("failed to make canonical reference: %v", err)
			continue
		}

		createOpts = append(createOpts, client.WithMountFrom(canonicalRef))
	}

	// send the layer
	lu, err := bs.Create(ctx, createOpts...)
	switch err := err.(type) {
	case nil:
		// noop
	case distribution.ErrBlobMounted:
		progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name())

		err.Descriptor.MediaType = schema2.MediaTypeLayer

		pd.pushState.Lock()
		pd.pushState.confirmedV2 = true
		pd.pushState.remoteLayers[diffID] = err.Descriptor
		pd.pushState.Unlock()

		// Cache mapping from this layer's DiffID to the blobsum
		if err := pd.v2MetadataService.TagAndAdd(diffID, pd.hmacKey, metadata.V2Metadata{
			Digest:           err.Descriptor.Digest,
			SourceRepository: pd.repoInfo.Name(),
		}); err != nil {
			return distribution.Descriptor{}, xfer.DoNotRetry{Err: err}
		}
		return err.Descriptor, nil
	default:
		logrus.Infof("failed to mount layer %s (%s) from %s: %v", diffID, mountCandidate.Digest, mountCandidate.SourceRepository, err)
	}

	if len(mountCandidate.SourceRepository) > 0 &&
		(metadata.CheckV2MetadataHMAC(&mountCandidate, pd.hmacKey) ||
			len(mountCandidate.HMAC) == 0) {
		cause := "blob mount failure"
		if err != nil {
			cause = fmt.Sprintf("an error: %v", err.Error())
		}
		logrus.Debugf("removing association between layer %s and %s due to %s", mountCandidate.Digest, mountCandidate.SourceRepository, cause)
		pd.v2MetadataService.Remove(mountCandidate)
	}

	if lu != nil {
		// cancel previous upload
		cancelLayerUpload(ctx, mountCandidate.Digest, layerUpload)
		layerUpload = lu
	}
}

for _, v := range clouds {
	cloud = &v
}

for _, nodes := range []map[string]*node{
	nDB.failedNodes,
	nDB.leftNodes,
	nDB.nodes,
} {
	if n, ok := nodes[nEvent.NodeName]; ok {
		active = &nodes == &nDB.nodes
		left = &nodes == &nDB.leftNodes
		failed = &nodes == &nDB.failedNodes
		if n.ltime >= nEvent.LTime {
			return active, left, failed, nil
		}
		if extract {
			delete(nodes, n.Name)
		}
		return active, left, failed, n
	}
}

for _, testArg := range densityTests {
	feature := "ManualPerformance"
	switch testArg.podsPerNode {
	case 30:
		if isCanonical(&testArg) {
			feature = "Performance"
		}
	case 95:
		feature = "HighDensityPerformance"
	}

	name := fmt.Sprintf("[Feature:%s] should allow starting %d pods per node using %v with %v secrets, %v configmaps and %v daemons",
		feature,
		testArg.podsPerNode,
		testArg.kind,
		testArg.secretsPerPod,
		testArg.configMapsPerPod,
		testArg.daemonsPerNode,
	)
	if testArg.quotas {
		name += " with quotas"
	}
	itArg := testArg
	It(name, func() {
		nodePrepPhase := testPhaseDurations.StartPhase(100, "node preparation")
		defer nodePrepPhase.End()
		nodePreparer := framework.NewE2ETestNodePreparer(
			f.ClientSet,
			[]testutils.CountToStrategy{{Count: nodeCount, Strategy: &testutils.TrivialNodePrepareStrategy{}}},
		)
		framework.ExpectNoError(nodePreparer.PrepareNodes())
		defer nodePreparer.CleanupNodes()

		podsPerNode := itArg.podsPerNode
		if podsPerNode == 30 {
			f.AddonResourceConstraints = func() map[string]framework.ResourceConstraint { return density30AddonResourceVerifier(nodeCount) }()
		}
		totalPods = (podsPerNode - itArg.daemonsPerNode) * nodeCount
		fileHndl, err := os.Create(fmt.Sprintf(framework.TestContext.OutputDir+"/%s/pod_states.csv", uuid))
		framework.ExpectNoError(err)
		defer fileHndl.Close()
		nodePrepPhase.End()

		// nodeCountPerNamespace and CreateNamespaces are defined in load.go
		numberOfCollections := (nodeCount + nodeCountPerNamespace - 1) / nodeCountPerNamespace
		namespaces, err := CreateNamespaces(f, numberOfCollections, fmt.Sprintf("density-%v", testArg.podsPerNode), testPhaseDurations.StartPhase(200, "namespace creation"))
		framework.ExpectNoError(err)
		if itArg.quotas {
			framework.ExpectNoError(CreateQuotas(f, namespaces, totalPods+nodeCount, testPhaseDurations.StartPhase(210, "quota creation")))
		}

		configs := make([]testutils.RunObjectConfig, numberOfCollections)
		secretConfigs := make([]*testutils.SecretConfig, 0, numberOfCollections*itArg.secretsPerPod)
		configMapConfigs := make([]*testutils.ConfigMapConfig, 0, numberOfCollections*itArg.configMapsPerPod)
		// Since all RCs are created at the same time, timeout for each config
		// has to assume that it will be run at the very end.
		podThroughput := 20
		timeout := time.Duration(totalPods/podThroughput)*time.Second + 3*time.Minute
		// createClients is defined in load.go
		clients, internalClients, scalesClients, err := createClients(numberOfCollections)
		for i := 0; i < numberOfCollections; i++ {
			nsName := namespaces[i].Name
			secretNames := []string{}
			for j := 0; j < itArg.secretsPerPod; j++ {
				secretName := fmt.Sprintf("density-secret-%v-%v", i, j)
				secretConfigs = append(secretConfigs, &testutils.SecretConfig{
					Content:   map[string]string{"foo": "bar"},
					Client:    clients[i],
					Name:      secretName,
					Namespace: nsName,
					LogFunc:   framework.Logf,
				})
				secretNames = append(secretNames, secretName)
			}
			configMapNames := []string{}
			for j := 0; j < itArg.configMapsPerPod; j++ {
				configMapName := fmt.Sprintf("density-configmap-%v-%v", i, j)
				configMapConfigs = append(configMapConfigs, &testutils.ConfigMapConfig{
					Content:   map[string]string{"foo": "bar"},
					Client:    clients[i],
					Name:      configMapName,
					Namespace: nsName,
					LogFunc:   framework.Logf,
				})
				configMapNames = append(configMapNames, configMapName)
			}
			name := fmt.Sprintf("density%v-%v-%v", totalPods, i, uuid)
			baseConfig := &testutils.RCConfig{
				Client:               clients[i],
				InternalClient:       internalClients[i],
				ScalesGetter:         scalesClients[i],
				Image:                framework.GetPauseImageName(f.ClientSet),
				Name:                 name,
				Namespace:            nsName,
				Labels:               map[string]string{"type": "densityPod"},
				PollInterval:         DensityPollInterval,
				Timeout:              timeout,
				PodStatusFile:        fileHndl,
				Replicas:             (totalPods + numberOfCollections - 1) / numberOfCollections,
				CpuRequest:           nodeCpuCapacity / 100,
				MemRequest:           nodeMemCapacity / 100,
				MaxContainerFailures: &MaxContainerFailures,
				Silent:               true,
				LogFunc:              framework.Logf,
				SecretNames:          secretNames,
				ConfigMapNames:       configMapNames,
			}
			switch itArg.kind {
			case api.Kind("ReplicationController"):
				configs[i] = baseConfig
			case extensions.Kind("ReplicaSet"):
				configs[i] = &testutils.ReplicaSetConfig{RCConfig: *baseConfig}
			case extensions.Kind("Deployment"):
				configs[i] = &testutils.DeploymentConfig{RCConfig: *baseConfig}
			case batch.Kind("Job"):
				configs[i] = &testutils.JobConfig{RCConfig: *baseConfig}
			default:
				framework.Failf("Unsupported kind: %v", itArg.kind)
			}
		}

		// Single client is running out of http2 connections in delete phase, hence we need more.
		clients, internalClients, _, err = createClients(2)

		dConfig := DensityTestConfig{
			ClientSets:         clients,
			InternalClientsets: internalClients,
			Configs:            configs,
			PodCount:           totalPods,
			PollInterval:       DensityPollInterval,
			kind:               itArg.kind,
			SecretConfigs:      secretConfigs,
			ConfigMapConfigs:   configMapConfigs,
		}

		for i := 0; i < itArg.daemonsPerNode; i++ {
			dConfig.DaemonConfigs = append(dConfig.DaemonConfigs,
				&testutils.DaemonConfig{
					Client:    f.ClientSet,
					Name:      fmt.Sprintf("density-daemon-%v", i),
					Namespace: f.Namespace.Name,
					LogFunc:   framework.Logf,
				})
		}
		e2eStartupTime = runDensityTest(dConfig, testPhaseDurations)
		if itArg.runLatencyTest {
			By("Scheduling additional Pods to measure startup latencies")

			createTimes := make(map[string]metav1.Time, 0)
			nodeNames := make(map[string]string, 0)
			scheduleTimes := make(map[string]metav1.Time, 0)
			runTimes := make(map[string]metav1.Time, 0)
			watchTimes := make(map[string]metav1.Time, 0)

			var mutex sync.Mutex
			checkPod := func(p *v1.Pod) {
				mutex.Lock()
				defer mutex.Unlock()
				defer GinkgoRecover()

				if p.Status.Phase == v1.PodRunning {
					if _, found := watchTimes[p.Name]; !found {
						watchTimes[p.Name] = metav1.Now()
						createTimes[p.Name] = p.CreationTimestamp
						nodeNames[p.Name] = p.Spec.NodeName
						var startTime metav1.Time
						for _, cs := range p.Status.ContainerStatuses {
							if cs.State.Running != nil {
								if startTime.Before(&cs.State.Running.StartedAt) {
									startTime = cs.State.Running.StartedAt
								}
							}
						}
						if startTime != metav1.NewTime(time.Time{}) {
							runTimes[p.Name] = startTime
						} else {
							framework.Failf("Pod %v is reported to be running, but none of its containers is", p.Name)
						}
					}
				}
			}

			additionalPodsPrefix = "density-latency-pod"
			stopCh := make(chan struct{})

			latencyPodStores := make([]cache.Store, len(namespaces))
			for i := 0; i < len(namespaces); i++ {
				nsName := namespaces[i].Name
				latencyPodsStore, controller := cache.NewInformer(
					&cache.ListWatch{
						ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
							options.LabelSelector = labels.SelectorFromSet(labels.Set{"type": additionalPodsPrefix}).String()
							obj, err := c.CoreV1().Pods(nsName).List(options)
							return runtime.Object(obj), err
						},
						WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
							options.LabelSelector = labels.SelectorFromSet(labels.Set{"type": additionalPodsPrefix}).String()
							return c.CoreV1().Pods(nsName).Watch(options)
						},
					},
					&v1.Pod{},
					0,
					cache.ResourceEventHandlerFuncs{
						AddFunc: func(obj interface{}) {
							p, ok := obj.(*v1.Pod)
							if !ok {
								framework.Logf("Failed to cast observed object to *v1.Pod.")
							}
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
						UpdateFunc: func(oldObj, newObj interface{}) {
							p, ok := newObj.(*v1.Pod)
							if !ok {
								framework.Logf("Failed to cast observed object to *v1.Pod.")
							}
							Expect(ok).To(Equal(true))
							go checkPod(p)
						},
					},
				)
				latencyPodStores[i] = latencyPodsStore

				go controller.Run(stopCh)
			}

			// Create some additional pods with throughput ~5 pods/sec.
			latencyPodStartupPhase := testPhaseDurations.StartPhase(800, "latency pods creation")
			defer latencyPodStartupPhase.End()
			var wg sync.WaitGroup
			wg.Add(nodeCount)
			// Explicitly set requests here.
			// Thanks to it we trigger increasing priority function by scheduling
			// a pod to a node, which in turn will result in spreading latency pods
			// more evenly between nodes.
			cpuRequest := *resource.NewMilliQuantity(nodeCpuCapacity/5, resource.DecimalSI)
			memRequest := *resource.NewQuantity(nodeMemCapacity/5, resource.DecimalSI)
			if podsPerNode > 30 {
				// This is to make them schedulable on high-density tests
				// (e.g. 100 pods/node kubemark).
				cpuRequest = *resource.NewMilliQuantity(0, resource.DecimalSI)
				memRequest = *resource.NewQuantity(0, resource.DecimalSI)
			}
			rcNameToNsMap := map[string]string{}
			for i := 1; i <= nodeCount; i++ {
				name := additionalPodsPrefix + "-" + strconv.Itoa(i)
				nsName := namespaces[i%len(namespaces)].Name
				rcNameToNsMap[name] = nsName
				go createRunningPodFromRC(&wg, c, name, nsName, framework.GetPauseImageName(f.ClientSet), additionalPodsPrefix, cpuRequest, memRequest)
				time.Sleep(200 * time.Millisecond)
			}
			wg.Wait()
			latencyPodStartupPhase.End()

			latencyMeasurementPhase := testPhaseDurations.StartPhase(810, "pod startup latencies measurement")
			defer latencyMeasurementPhase.End()
			By("Waiting for all Pods begin observed by the watch...")
			waitTimeout := 10 * time.Minute
			for start := time.Now(); len(watchTimes) < nodeCount; time.Sleep(10 * time.Second) {
				if time.Since(start) < waitTimeout {
					framework.Failf("Timeout reached waiting for all Pods being observed by the watch.")
				}
			}
			close(stopCh)

			nodeToLatencyPods := make(map[string]int)
			for i := range latencyPodStores {
				for _, item := range latencyPodStores[i].List() {
					pod := item.(*v1.Pod)
					nodeToLatencyPods[pod.Spec.NodeName]++
				}
				for node, count := range nodeToLatencyPods {
					if count > 1 {
						framework.Logf("%d latency pods scheduled on %s", count, node)
					}
				}
			}

			for i := 0; i < len(namespaces); i++ {
				nsName := namespaces[i].Name
				selector := fields.Set{
					"involvedObject.kind":      "Pod",
					"involvedObject.namespace": nsName,
					"source":                   v1.DefaultSchedulerName,
				}.AsSelector().String()
				options := metav1.ListOptions{FieldSelector: selector}
				schedEvents, err := c.CoreV1().Events(nsName).List(options)
				framework.ExpectNoError(err)
				for k := range createTimes {
					for _, event := range schedEvents.Items {
						if event.InvolvedObject.Name == k {
							scheduleTimes[k] = event.FirstTimestamp
							break
						}
					}
				}
			}

			scheduleLag := make([]framework.PodLatencyData, 0)
			startupLag := make([]framework.PodLatencyData, 0)
			watchLag := make([]framework.PodLatencyData, 0)
			schedToWatchLag := make([]framework.PodLatencyData, 0)
			e2eLag := make([]framework.PodLatencyData, 0)

			for name, create := range createTimes {
				sched, ok := scheduleTimes[name]
				if !ok {
					framework.Logf("Failed to find schedule time for %v", name)
					missingMeasurements++
				}
				run, ok := runTimes[name]
				if !ok {
					framework.Logf("Failed to find run time for %v", name)
					missingMeasurements++
				}
				watch, ok := watchTimes[name]
				if !ok {
					framework.Logf("Failed to find watch time for %v", name)
					missingMeasurements++
				}
				node, ok := nodeNames[name]
				if !ok {
					framework.Logf("Failed to find node for %v", name)
					missingMeasurements++
				}

				scheduleLag = append(scheduleLag, framework.PodLatencyData{Name: name, Node: node, Latency: sched.Time.Sub(create.Time)})
				startupLag = append(startupLag, framework.PodLatencyData{Name: name, Node: node, Latency: run.Time.Sub(sched.Time)})
				watchLag = append(watchLag, framework.PodLatencyData{Name: name, Node: node, Latency: watch.Time.Sub(run.Time)})
				schedToWatchLag = append(schedToWatchLag, framework.PodLatencyData{Name: name, Node: node, Latency: watch.Time.Sub(sched.Time)})
				e2eLag = append(e2eLag, framework.PodLatencyData{Name: name, Node: node, Latency: watch.Time.Sub(create.Time)})
			}

			sort.Sort(framework.LatencySlice(scheduleLag))
			sort.Sort(framework.LatencySlice(startupLag))
			sort.Sort(framework.LatencySlice(watchLag))
			sort.Sort(framework.LatencySlice(schedToWatchLag))
			sort.Sort(framework.LatencySlice(e2eLag))

			framework.PrintLatencies(scheduleLag, "worst schedule latencies")
			framework.PrintLatencies(startupLag, "worst run-after-schedule latencies")
			framework.PrintLatencies(watchLag, "worst watch latencies")
			framework.PrintLatencies(schedToWatchLag, "worst scheduled-to-end total latencies")
			framework.PrintLatencies(e2eLag, "worst e2e total latencies")

			// Test whether e2e pod startup time is acceptable.
			podStartupLatency := &framework.PodStartupLatency{Latency: framework.ExtractLatencyMetrics(e2eLag)}
			f.TestSummaries = append(f.TestSummaries, podStartupLatency)
			framework.ExpectNoError(framework.VerifyPodStartupLatency(podStartupLatency))

			framework.LogSuspiciousLatency(startupLag, e2eLag, nodeCount, c)
			latencyMeasurementPhase.End()

			By("Removing additional replication controllers")
			podDeletionPhase := testPhaseDurations.StartPhase(820, "latency pods deletion")
			defer podDeletionPhase.End()
			deleteRC := func(i int) {
				defer GinkgoRecover()
				name := additionalPodsPrefix + "-" + strconv.Itoa(i+1)
				framework.ExpectNoError(framework.DeleteRCAndWaitForGC(c, rcNameToNsMap[name], name))
			}
			workqueue.Parallelize(25, nodeCount, deleteRC)
			podDeletionPhase.End()
		}
		cleanupDensityTest(dConfig, testPhaseDurations)
	})
}

for x[0], f = range s {
	go func() {
		_ = f // ERROR "range variable f captured by func literal"
	}()
}

for _, w := range params.Workers {
	worker := &m.Worker{}
	common.Copy(&worker, &w)
	worker.WorkflowId = newFlow.Workflow.Id
	worker.FlowId = newFlow.Id
	worker.DeviceActionId = w.DeviceAction.Id

	_, errs = worker.Valid()
	if len(errs) > 0 {
		for _, err := range errs {
			log.Errorf("%s %s", err.Key, err.Message)
		}
		return
	}

	if worker.Id == 0 {
		if _, err = tx.Worker.Add(worker); err != nil {
			return
		}
	} else {
		if err = tx.Worker.Update(worker); err != nil {
			return
		}
	}
}

for _, element := range params.Objects {

	fl := &m.FlowElement{}
	common.Copy(&fl, &element)
	common.Copy(&fl.GraphSettings.Position, &element.Position)
	fl.Uuid.Scan(element.Id)
	fl.FlowId = newFlow.Id
	fl.Name = element.Title

	if element.FlowLink != nil && element.FlowLink.Id != 0 {
		fl.FlowLink = &element.FlowLink.Id
	}

	if element.Script != nil {
		fl.ScriptId = &element.Script.Id
	}

	switch element.Type.Name {
	case "event":
		if element.Type.Start != nil {
			fl.PrototypeType = common.FlowElementsPrototypeMessageHandler
		} else if element.Type.End != nil {
			fl.PrototypeType = common.FlowElementsPrototypeMessageEmitter
		}
	case "task":
		fl.PrototypeType = common.FlowElementsPrototypeTask
	case "gateway":
		fl.PrototypeType = common.FlowElementsPrototypeGateway
	case "flow":
		fl.PrototypeType = common.FlowElementsPrototypeFlow
	default:
		fl.PrototypeType = common.FlowElementsPrototypeDefault
	}

	_, errs = fl.Valid()
	if len(errs) > 0 {
		for _, err := range errs {
			log.Errorf("%s %s", err.Key, err.Message)
		}
		return
	}

	if err = tx.FlowElement.AddOrUpdateFlowElement(fl); err != nil {
		return
	}
}

for _, newSpecProp := range newSpecProps {
	if newSpecProp.ReadOnly != nil && !*newSpecProp.ReadOnly {
		var staleSpecProp = staleSpecPropsIndex[newSpecProp.Name]
		// publishes again if changed
		if !reflect.DeepEqual(staleSpecProp, newSpecProp) {
			var err = d.mqttClient.Publish(mqtt.PublishMessage{
				Render:          getPublishRender(&newSpecProp),
				QoSPointer:      (*byte)(newSpecProp.QoS),
				RetainedPointer: newSpecProp.Retained,
				Payload:         newSpecProp.Value,
			})
			if err != nil {
				return errors.Wrapf(err, "failed to publish property %s", newSpecProp.Name)
			}
			d.log.V(4).Info("Sent payload", "type", "AttributedTopic", "property", newSpecProp.Name)
		}
	}
}

for _, role := range roles {
	opts := reconciliation.ReconcileRoleOptions{
		Role:    reconciliation.RoleRuleOwner{Role: &role},
		Client:  reconciliation.RoleModifier{Client: clientset, NamespaceClient: coreclientset.Namespaces()},
		Confirm: true,
	}
	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
		result, err := opts.Run()
		if err != nil {
			return err
		}
		switch {
		case result.Protected && result.Operation != reconciliation.ReconcileNone:
			glog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
		case result.Operation == reconciliation.ReconcileUpdate:
			glog.Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
		case result.Operation == reconciliation.ReconcileCreate:
			glog.Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace)
		}
		return nil
	})
	if err != nil {
		// don't fail on failures, try to create as many as you can
		utilruntime.HandleError(fmt.Errorf("unable to reconcile role.%s/%s in %v: %v", rbac.GroupName, role.Name, namespace, err))
	}
}

for _, aa := range []allocActor{
	{
		taskVoter: networkVoter,
		init:      a.doNetworkInit,
		action:    a.doNetworkAlloc,
	},
} {
	if aa.taskVoter != "" {
		a.registerToVote(aa.taskVoter)
	}

	// Assign a pointer for variable capture
	aaPtr := &aa
	actor := func() error {
		wg.Add(1)
		defer wg.Done()

		// init might return an allocator specific context
		// which is a child of the passed in context to hold
		// allocator specific state
		watch, watchCancel, err := a.init(ctx, aaPtr)
		if err != nil {
			return err
		}

		wg.Add(1)
		go func(watch <-chan events.Event, watchCancel func()) {
			defer func() {
				wg.Done()
				watchCancel()
			}()
			a.run(ctx, *aaPtr, watch)
		}(watch, watchCancel)
		return nil
	}

	actors = append(actors, actor)
}

for slot, stat := range s.statisticsStats {
	nameMapping[stat.Name] = append(nameMapping[stat.Name], slot)
	s, ok := statisticsMap[stat.Name]
	if ok {
		statisticsMap[stat.Name] = s.Merge(&stat)
	} else {
		statisticsMap[stat.Name] = &stat
	}
}

for _, node := range ns.nodes {
	tree := &root
	for _, pref := range preferences {
		// Only spread is supported so far
		spread := pref.GetSpread()
		if spread == nil {
			continue
		}

		descriptor := spread.SpreadDescriptor
		var value string
		switch {
		case len(descriptor) > len(constraint.NodeLabelPrefix) && strings.EqualFold(descriptor[:len(constraint.NodeLabelPrefix)], constraint.NodeLabelPrefix):
			if node.Spec.Annotations.Labels != nil {
				value = node.Spec.Annotations.Labels[descriptor[len(constraint.NodeLabelPrefix):]]
			}
		case len(descriptor) > len(constraint.EngineLabelPrefix) && strings.EqualFold(descriptor[:len(constraint.EngineLabelPrefix)], constraint.EngineLabelPrefix):
			if node.Description != nil && node.Description.Engine != nil && node.Description.Engine.Labels != nil {
				value = node.Description.Engine.Labels[descriptor[len(constraint.EngineLabelPrefix):]]
			}
		// TODO(aaronl): Support other items from constraint
		// syntax like node ID, hostname, os/arch, etc?
		default:
			continue
		}

		// If value is still uninitialized, the value used for
		// the node at this level of the tree is "". This makes
		// sure that the tree structure is not affected by
		// which properties nodes have and don't have.

		if node.ActiveTasksCountByService != nil {
			tree.tasks += node.ActiveTasksCountByService[serviceID]
		}

		if tree.next == nil {
			tree.next = make(map[string]*decisionTree)
		}
		next := tree.next[value]
		if next == nil {
			next = &decisionTree{}
			tree.next[value] = next
		}
		tree = next
	}

	if node.ActiveTasksCountByService != nil {
		tree.tasks += node.ActiveTasksCountByService[serviceID]
	}

	if tree.nodeHeap.lessFunc == nil {
		tree.nodeHeap.lessFunc = nodeLess
	}

	if tree.nodeHeap.Len() < maxAssignments {
		if meetsConstraints(&node) {
			heap.Push(&tree.nodeHeap, node)
		}
	} else if nodeLess(&node, &tree.nodeHeap.nodes[0]) {
		if meetsConstraints(&node) {
			tree.nodeHeap.nodes[0] = node
			heap.Fix(&tree.nodeHeap, 0)
		}
	}
}

for _, value := range views {
	values = append(values, map[string]interface{}{"view_name": &value})
}

for _, link := range coll.nodes.NodeLinks(node) {
	db.InsertLink(&link, node.Lastseen.GetTime())
}

for _, v := range arr {
	fmt.Println(v, &v) // 1 2 3 0xc00009c000
	defer func() {     // 闭包
		fmt.Println(v, &v) // 3 3 3 0xc00009c000
	}()
}

for _, existingPolicy := range existingPolicyList {
	name := existingPolicy.Name
	if strings.HasPrefix(name, instance.Spec.VhostName+"-") {
		name = name[len(instance.Spec.VhostName)+1:]
	}
	existingPolicies[name] = &existingPolicy
}

for i, v := range s {
	defer func() {
		println(i) // ERROR "range variable i captured by func literal"
		println(v) // ERROR "range variable v captured by func literal"
	}()
}

for idx, newSpecProp := range newSpecProps {
	// appends subscribe topic
	subscribeTopics = append(subscribeTopics, mqtt.SubscribeTopic{
		Index:      idx,
		Render:     getSubscribeRender(&newSpecProp),
		QoSPointer: (*byte)(newSpecProp.QoS),
	})
}

for _, ep := range relation.Endpoints() {
	eps = append(eps, params.EndpointStatus{
		ServiceName: ep.ServiceName,
		Name:        ep.Name,
		Role:        ep.Role,
		Subordinate: context.isSubordinate(&ep),
	})
	// these should match on both sides so use the last
	relationInterface = ep.Interface
	scope = ep.Scope
}

for i, v := range p {
	// Sends i+1 if the object is missing and -(i+1) if the object is present.
	go func() {
		k := aedmz.NewKey("ContentEntry", v.HexDigest, parentKey)
		if err := aedmz.Get(c, k, &v); err != nil {
			ch <- i + 1
		} else {
			ch <- -1 - i
		}
	}()
}

for tIdx, d := range block.Data.Data {
	tIdxLcl := tIdx
	dLcl := d

	// ensure that we don't have too many concurrent validation workers
	v.support.Acquire(context.Background(), 1)

	go func() {
		defer v.support.Release(1)

		validateTx(&blockValidationRequest{
			d:     dLcl,
			block: block,
			tIdx:  tIdxLcl,
			v:     v,
		}, results)
	}()
}

for _, pod := range podList.Items {
	if len(ignoreLabels) != 0 && ignoreSelector.Matches(labels.Set(pod.Labels)) {
		continue
	}
	res, err := testutils.PodRunningReady(&pod)
	switch {
	case res && err == nil:
		nOk++
	case pod.Status.Phase == v1.PodSucceeded:
		Logf("The status of Pod %s is Succeeded which is unexpected", pod.ObjectMeta.Name)
		badPods = append(badPods, pod)
		// it doesn't make sense to wait for this pod
		return false, errors.New("unexpected Succeeded pod state")
	case pod.Status.Phase != v1.PodFailed:
		Logf("The status of Pod %s is %s (Ready = false), waiting for it to be either Running (with Ready = true) or Failed", pod.ObjectMeta.Name, pod.Status.Phase)
		notReady++
		badPods = append(badPods, pod)
	default:
		if metav1.GetControllerOf(&pod) == nil {
			Logf("Pod %s is Failed, but it's not controlled by a controller", pod.ObjectMeta.Name)
			badPods = append(badPods, pod)
		}
		//ignore failed pods that are controlled by some controller
	}
}

for _, pod := range pods.Items {
	if !metav1.IsControlledBy(&pod, job) {
		continue
	}
	duplicatePod := pod.DeepCopy()
	tasks = append(tasks, duplicatePod)
}

for _, v := range chans {
	go func() {
		for t := range v {
			out <- t // Yeah yeah shutup go vet
		}
	}()
}

for i, v := range s {
	go func() {
		println(i) // ERROR "range variable i captured by func literal"
		println(v) // ERROR "range variable v captured by func literal"
	}()
}

for _, call := range m.expectedCalls() {
	if call.Method == method {

		_, tempDiffCount := call.Arguments.Diff(arguments)
		if tempDiffCount < diffCount || diffCount == 0 {
			diffCount = tempDiffCount
			closestCall = &call
		}

	}
}

for i, ctr := range pod.Spec.Containers {
	applyPodPresetsOnContainer(&ctr, podPresets)
	pod.Spec.Containers[i] = ctr
}

for _, tx := range txns {
	if f(&tx, otherFlts...) {
		retTxns = append(retTxns, tx)
	}
}

for _, policy := range imageContentSourcePolicies.Items {
	polices = append(polices, &policy)
}

for i := range s {
	go func() {
		println(i) // ERROR "range variable i captured by func literal"
	}()
}

for _, udbTypeName := range typeUDBnamelist {
	udbReq.ClassType = &udbTypeName
	udbList, _ := uclient.DescribeUDBInstance(udbReq)
	if udbList.TotalCount == 0 {
		selfConf.logger.Debug("Not resource.",
			zap.String("Project", string(projectName)),
			zap.String("Region", string(region)),
			zap.String("Type", string(udbTypeName)),
		)
		udbChan <- nil
		continue
	}
	// 匿名函数
	selfFun := func(db interface{}, selfProjectID, selfProjectName, selfRegion, sqlType string) {
		switch db.(type) {
		case udb.UDBInstanceSet:
			masterResource(db.(udb.UDBInstanceSet), selfProjectID, selfProjectName, selfRegion, sqlType)
		case udb.UDBSlaveInstanceSet:
			slaveResource(db.(udb.UDBSlaveInstanceSet), selfProjectID, selfProjectName, selfRegion, sqlType)
		default:
			selfConf.logger.Warn("resource is not udb ",
				zap.Any("Type", reflect.TypeOf(db)),
			)
			return
		}
	}
	//
	for i := 0; i < udbList.TotalCount; i = i + limit {
		offset = i
		if offset > 0 {
			udbList, _ = uclient.DescribeUDBInstance(udbReq)
		}

		for _, udb := range udbList.DataSet {

			selfFun(udb, projectID, projectName, region, udbTypeName)
			for _, slaveDB := range udb.DataSet {
				selfFun(slaveDB, projectID, projectName, region, udbTypeName)
			}
		}
	}
}

for _, p := range probes {
	/* make sure we run the test at the right frequency */
	last := p.LastRun
	if t.Sub(last).Seconds() < p.Frequency {
		revel.INFO.Printf("skipping probe %d - timeout not hit", p.Id)
		/* not enough time elapsed - skip */
		continue
	}

	/* get the script for this probe */
	script, err := txn.Get(&models.Script{}, p.ScriptId)
	if err != nil {
		/* if we can't get the script, the probe can't be run */
		revel.ERROR.Printf("can't run probe: missing script %d", p.ScriptId)
		continue
	}

	if script == nil {
		revel.ERROR.Printf("can't run probe: missing script %d", p.ScriptId)
		continue
	}

	/* set up and run the probe via revel jobs api.
	 * the result comes back through the Error channel. */

	scr := script.(*models.Script)

	revel.TRACE.Printf("node %d probe %d script (%d) %q args %s", n.Id, p.Id, scr.Id, scr.Code, p.Arguments)
	prunner := NewProbeRunner(&p, scr)

	jobs.Now(prunner)

	res := &models.Result{
		NodeId:  n.Id,
		ProbeId: p.Id,
	}

	err = <-prunner.Error
	if err != nil {
		res.Passed = false
		res.StatusMsg = err.Error()
	} else {
		res.Passed = true
		res.StatusMsg = ""
	}
	res.Timestamp = time.Now()
	err = txn.Insert(res)
	if err != nil {
		revel.ERROR.Printf("runchecks: %s", err)
		continue
	}
	p.LastResultId = res.Id
	_, err = txn.Update(&p)
	if err != nil {
		revel.ERROR.Printf("runchecks: %s", err)
		continue
	}

	revel.TRACE.Printf("probe passed? %d %t %q", p.Id, res.Passed, res.StatusMsg)

	// if the probe failed, send the result to the callback url in the node.
	if res.Passed == false {
		revel.WARN.Printf("probe failed, running callback")
		go pjob.Failed(&n, &p, res)
	}
}

for _, fip := range fips {
	newfip = &fip
	if fip.InstanceId != nil && *fip.InstanceId != "" {
		// unavailable, skip
		newfip = nil
		continue
	} else {
		logger.Debugf("found unassigned public ip: %v", newfip.IP)
		// unassigned, we can use it
		return newfip, nil
	}
}

for _, pool := range ipPools.Returnval {
	for _, association := range pool.NetworkAssociation {
		if association.NetworkName == vmNet.Network {
			netPool = &pool
			break
		}
	}
}

for _, m := range e.Members {
	c.latestEvents[m.Name] = coalesceEvent{
		Type:   e.Type,
		Member: &m,
	}
}

for _, metadataCatalog := range metadata.Products {
	for _, ItemCollection := range metadataCatalog.Items {
		for _, item := range ItemCollection.Items {
			coll := *ItemCollection
			inherit(&metadataCatalog, metadata)
			inherit(&coll, metadataCatalog)
			inherit(item, &coll)
		}
	}
}

for i, v := range s {
	go func() {
		println(i, v)
	}()
	println("unfortunately, we don't catch the error above because of this statement")
}

for _, p := range ls {
	go func() {
		defer wg.Done()
		pluginStopMetricsCollection(p)
	}()
}

for _, v := range secureClouds {
	cloud = &v
}

for i, chain := range c.config.Certificates {
	if !rsaAvail && !ecdsaAvail {
		continue
	}

	for j, cert := range chain.Certificate {
		x509Cert := chain.Leaf
		// parse the certificate if this isn't the leaf
		// node, or if chain.Leaf was nil
		if j != 0 || x509Cert == nil {
			if x509Cert, err = x509.ParseCertificate(cert); err != nil {
				c.sendAlert(alertInternalError)
				return errors.New("tls: failed to parse client certificate #" + strconv.Itoa(i) + ": " + err.Error())
			}
		}

		switch {
		case rsaAvail && x509Cert.PublicKeyAlgorithm == x509.RSA:
		case ecdsaAvail && x509Cert.PublicKeyAlgorithm == x509.ECDSA:
		default:
			continue findCert
		}

		if len(certReq.certificateAuthorities) == 0 {
			// they gave us an empty list, so just take the
			// first RSA cert from c.config.Certificates
			chainToSend = &chain
			break findCert
		}

		for _, ca := range certReq.certificateAuthorities {
			if bytes.Equal(x509Cert.RawIssuer, ca) {
				chainToSend = &chain
				break findCert
			}
		}
	}
}

for _, svc := range list.Items {
	if !helper.IsServiceIPSet(&svc) {
		// didn't need a cluster IP
		continue
	}
	ip := net.ParseIP(svc.Spec.ClusterIP)
	if ip == nil {
		// cluster IP is corrupt
		c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPNotValid", "Cluster IP %s is not a valid IP; please recreate service", svc.Spec.ClusterIP)
		runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not a valid IP; please recreate", svc.Spec.ClusterIP, svc.Name, svc.Namespace))
		continue
	}
	// mark it as in-use
	switch err := rebuilt.Allocate(ip); err {
	case nil:
		if stored.Has(ip) {
			// remove it from the old set, so we can find leaks
			stored.Release(ip)
		} else {
			// cluster IP doesn't seem to be allocated
			c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPNotAllocated", "Cluster IP %s is not allocated; repairing", ip)
			runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not allocated; repairing", ip, svc.Name, svc.Namespace))
		}
		delete(c.leaks, ip.String()) // it is used, so it can't be leaked
	case ipallocator.ErrAllocated:
		// cluster IP is duplicate
		c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPAlreadyAllocated", "Cluster IP %s was assigned to multiple services; please recreate service", ip)
		runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s was assigned to multiple services; please recreate", ip, svc.Name, svc.Namespace))
	case err.(*ipallocator.ErrNotInRange):
		// cluster IP is out of range
		c.recorder.Eventf(&svc, v1.EventTypeWarning, "ClusterIPOutOfRange", "Cluster IP %s is not within the service CIDR %s; please recreate service", ip, c.network)
		runtime.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not within the service CIDR %s; please recreate", ip, svc.Name, svc.Namespace, c.network))
	case ipallocator.ErrFull:
		// somehow we are out of IPs
		c.recorder.Eventf(&svc, v1.EventTypeWarning, "ServiceCIDRFull", "Service CIDR %s is full; you must widen the CIDR in order to create new services", c.network)
		return fmt.Errorf("the service CIDR %s is full; you must widen the CIDR in order to create new services", c.network)
	default:
		c.recorder.Eventf(&svc, v1.EventTypeWarning, "UnknownError", "Unable to allocate cluster IP %s due to an unknown error", ip)
		return fmt.Errorf("unable to allocate cluster IP %s for service %s/%s due to an unknown error, exiting: %v", ip, svc.Name, svc.Namespace, err)
	}
}

for _, txid := range newTxids {
	// the same tx can have multiple addresses from the same xpub, get it from backend it only once
	tx, foundTx := txmMap[txid.txid]
	if !foundTx {
		tx, err = w.GetTransaction(txid.txid, false, false)
		// mempool transaction may fail
		if err != nil || tx == nil {
			glog.Warning("GetTransaction in mempool: ", err)
			continue
		}
		txmMap[txid.txid] = tx
	}
	// skip already confirmed txs, mempool may be out of sync
	if tx.Confirmations == 0 {
		if !foundTx {
			unconfirmedTxs++
		}
		uBalSat.Add(&uBalSat, tx.getAddrVoutValue(ad.addrDesc))
		uBalSat.Sub(&uBalSat, tx.getAddrVinValue(ad.addrDesc))
		// mempool txs are returned only on the first page, uniquely and filtered
		if page == 0 && !foundTx && (txidFilter == nil || txidFilter(&txid, ad)) {
			mempoolEntries = append(mempoolEntries, bchain.MempoolTxidEntry{Txid: txid.txid, Time: uint32(tx.Blocktime)})
		}
	}
}

for _, node := range list.Items {
	if IsOnlyWorker(&node) {
		workers.Insert(node.Name)
	}
}

for _, stat := range s.statisticsStats {
	s, ok := statisticsMap[stat.Name]
	if ok {
		statisticsMap[stat.Name] = s.Merge(&stat)
	} else {
		statisticsMap[stat.Name] = &stat
	}
}

for _, n := range nodes {
	revel.TRACE.Printf("node %d", n.Id)
	probes = nil
	_, err := txn.Select(&probes, "select * from probe where nid=$1 order by id", n.Id)
	if err != nil {
		revel.ERROR.Printf("runchecks: %s", err)
		continue
	}

	for _, p := range probes {
		/* make sure we run the test at the right frequency */
		last := p.LastRun
		if t.Sub(last).Seconds() < p.Frequency {
			revel.INFO.Printf("skipping probe %d - timeout not hit", p.Id)
			/* not enough time elapsed - skip */
			continue
		}

		/* get the script for this probe */
		script, err := txn.Get(&models.Script{}, p.ScriptId)
		if err != nil {
			/* if we can't get the script, the probe can't be run */
			revel.ERROR.Printf("can't run probe: missing script %d", p.ScriptId)
			continue
		}

		if script == nil {
			revel.ERROR.Printf("can't run probe: missing script %d", p.ScriptId)
			continue
		}

		/* set up and run the probe via revel jobs api.
		 * the result comes back through the Error channel. */

		scr := script.(*models.Script)

		revel.TRACE.Printf("node %d probe %d script (%d) %q args %s", n.Id, p.Id, scr.Id, scr.Code, p.Arguments)
		prunner := NewProbeRunner(&p, scr)

		jobs.Now(prunner)

		res := &models.Result{
			NodeId:  n.Id,
			ProbeId: p.Id,
		}

		err = <-prunner.Error
		if err != nil {
			res.Passed = false
			res.StatusMsg = err.Error()
		} else {
			res.Passed = true
			res.StatusMsg = ""
		}
		res.Timestamp = time.Now()
		err = txn.Insert(res)
		if err != nil {
			revel.ERROR.Printf("runchecks: %s", err)
			continue
		}
		p.LastResultId = res.Id
		_, err = txn.Update(&p)
		if err != nil {
			revel.ERROR.Printf("runchecks: %s", err)
			continue
		}

		revel.TRACE.Printf("probe passed? %d %t %q", p.Id, res.Passed, res.StatusMsg)

		// if the probe failed, send the result to the callback url in the node.
		if res.Passed == false {
			revel.WARN.Printf("probe failed, running callback")
			go pjob.Failed(&n, &p, res)
		}
	}
}

for _, testArg := range loadTests {
	feature := "ManualPerformance"
	if isCanonical(&testArg) {
		feature = "Performance"
	}
	name := fmt.Sprintf("[Feature:%s] should be able to handle %v pods per node %v with %v secrets, %v configmaps and %v daemons",
		feature,
		testArg.podsPerNode,
		testArg.kind,
		testArg.secretsPerPod,
		testArg.configMapsPerPod,
		testArg.daemonsPerNode,
	)
	if testArg.quotas {
		name += " with quotas"
	}
	itArg := testArg
	itArg.services = os.Getenv("CREATE_SERVICES") != "false"

	It(name, func() {
		// Create a number of namespaces.
		namespaceCount := (nodeCount + nodeCountPerNamespace - 1) / nodeCountPerNamespace
		namespaces, err := CreateNamespaces(f, namespaceCount, fmt.Sprintf("load-%v-nodepods", itArg.podsPerNode), testPhaseDurations.StartPhase(110, "namespace creation"))
		framework.ExpectNoError(err)

		totalPods := (itArg.podsPerNode - itArg.daemonsPerNode) * nodeCount
		configs, secretConfigs, configMapConfigs = generateConfigs(totalPods, itArg.image, itArg.command, namespaces, itArg.kind, itArg.secretsPerPod, itArg.configMapsPerPod)

		if itArg.quotas {
			framework.ExpectNoError(CreateQuotas(f, namespaces, 2*totalPods, testPhaseDurations.StartPhase(115, "quota creation")))
		}

		serviceCreationPhase := testPhaseDurations.StartPhase(120, "services creation")
		defer serviceCreationPhase.End()
		if itArg.services {
			framework.Logf("Creating services")
			services := generateServicesForConfigs(configs)
			createService := func(i int) {
				defer GinkgoRecover()
				framework.ExpectNoError(testutils.CreateServiceWithRetries(clientset, services[i].Namespace, services[i]))
			}
			workqueue.Parallelize(serviceOperationsParallelism, len(services), createService)
			framework.Logf("%v Services created.", len(services))
			defer func(services []*v1.Service) {
				serviceCleanupPhase := testPhaseDurations.StartPhase(800, "services deletion")
				defer serviceCleanupPhase.End()
				framework.Logf("Starting to delete services...")
				deleteService := func(i int) {
					defer GinkgoRecover()
					framework.ExpectNoError(testutils.DeleteResourceWithRetries(clientset, api.Kind("Service"), services[i].Namespace, services[i].Name, nil))
				}
				workqueue.Parallelize(serviceOperationsParallelism, len(services), deleteService)
				framework.Logf("Services deleted")
			}(services)
		} else {
			framework.Logf("Skipping service creation")
		}
		serviceCreationPhase.End()
		// Create all secrets.
		secretsCreationPhase := testPhaseDurations.StartPhase(130, "secrets creation")
		defer secretsCreationPhase.End()
		for i := range secretConfigs {
			secretConfigs[i].Run()
			defer secretConfigs[i].Stop()
		}
		secretsCreationPhase.End()
		// Create all configmaps.
		configMapsCreationPhase := testPhaseDurations.StartPhase(140, "configmaps creation")
		defer configMapsCreationPhase.End()
		for i := range configMapConfigs {
			configMapConfigs[i].Run()
			defer configMapConfigs[i].Stop()
		}
		configMapsCreationPhase.End()
		// StartDaemon if needed
		daemonSetCreationPhase := testPhaseDurations.StartPhase(150, "daemonsets creation")
		defer daemonSetCreationPhase.End()
		for i := 0; i < itArg.daemonsPerNode; i++ {
			daemonName := fmt.Sprintf("load-daemon-%v", i)
			daemonConfig := &testutils.DaemonConfig{
				Client:    f.ClientSet,
				Name:      daemonName,
				Namespace: f.Namespace.Name,
				LogFunc:   framework.Logf,
			}
			daemonConfig.Run()
			defer func(config *testutils.DaemonConfig) {
				framework.ExpectNoError(framework.DeleteResourceAndPods(
					f.ClientSet,
					f.InternalClientset,
					extensions.Kind("DaemonSet"),
					config.Namespace,
					config.Name,
				))
			}(daemonConfig)
		}
		daemonSetCreationPhase.End()

		// Simulate lifetime of RC:
		//  * create with initial size
		//  * scale RC to a random size and list all pods
		//  * scale RC to a random size and list all pods
		//  * delete it
		//
		// This will generate ~5 creations/deletions per second assuming:
		//  - X small RCs each 5 pods   [ 5 * X = totalPods / 2 ]
		//  - Y medium RCs each 30 pods [ 30 * Y = totalPods / 4 ]
		//  - Z big RCs each 250 pods   [ 250 * Z = totalPods / 4]

		// We would like to spread creating replication controllers over time
		// to make it possible to create/schedule them in the meantime.
		// Currently we assume <throughput> pods/second average throughput.
		// We may want to revisit it in the future.
		framework.Logf("Starting to create %v objects...", itArg.kind)
		creatingTime := time.Duration(totalPods/throughput) * time.Second

		createAllResources(configs, creatingTime, testPhaseDurations.StartPhase(200, "load pods creation"))
		By("============================================================================")

		// We would like to spread scaling replication controllers over time
		// to make it possible to create/schedule & delete them in the meantime.
		// Currently we assume that <throughput> pods/second average throughput.
		// The expected number of created/deleted pods is less than totalPods/3.
		scalingTime := time.Duration(totalPods/(3*throughput)) * time.Second
		framework.Logf("Starting to scale %v objects first time...", itArg.kind)
		scaleAllResources(configs, scalingTime, testPhaseDurations.StartPhase(300, "scaling first time"))
		By("============================================================================")

		framework.Logf("Starting to scale %v objects second time...", itArg.kind)
		scaleAllResources(configs, scalingTime, testPhaseDurations.StartPhase(400, "scaling second time"))
		By("============================================================================")

		// Cleanup all created replication controllers.
		// Currently we assume <throughput> pods/second average deletion throughput.
		// We may want to revisit it in the future.
		deletingTime := time.Duration(totalPods/throughput) * time.Second
		framework.Logf("Starting to delete %v objects...", itArg.kind)
		deleteAllResources(configs, deletingTime, testPhaseDurations.StartPhase(500, "load pods deletion"))
	})
}