garbage collector controller 源码分析

$ kubectl delete ds/nginx-ds --cascade=false

$ curl -k -v -XDELETE  -H "Accept: application/json" -H "Content-Type: application/json" -d '{"propagationPolicy":"Foreground"}' 'https://192.168.99.108:8443/apis/apps/v1/namespaces/default/daemonsets/nginx-ds'

{
	......
	"metadata": {
	   ......
		"finalizers": [
			"foregroundDeletion"
		]
	}
	......
}

func startGarbageCollectorController(ctx ControllerContext) (http.Handler, bool, error) {
    if !ctx.ComponentConfig.GarbageCollectorController.EnableGarbageCollector {
        return nil, false, nil
    }
    // 1、初始化 discoveryClient
    gcClientset := ctx.ClientBuilder.ClientOrDie("generic-garbage-collector")
    discoveryClient := cacheddiscovery.NewMemCacheClient(gcClientset.Discovery())

    config := ctx.ClientBuilder.ConfigOrDie("generic-garbage-collector")
    metadataClient, err := metadata.NewForConfig(config)
    if err != nil {
        return nil, true, err
    }

    // 2、获取 deletableResource
    deletableResources := garbagecollector.GetDeletableResources(discoveryClient)
    ignoredResources := make(map[schema.GroupResource]struct{})
    for _, r := range ctx.ComponentConfig.GarbageCollectorController.GCIgnoredResources {
        ignoredResources[schema.GroupResource{Group: r.Group, Resource: r.Resource}] = struct{}{}
    }
    
    // 3、初始化 garbageCollector 对象
    garbageCollector, err := garbagecollector.NewGarbageCollector(
        ......
    )
    if err != nil {
        return nil, true, fmt.Errorf("failed to start the generic garbage collector: %v", err)
    }
    // 4、启动 garbage collector
    workers := int(ctx.ComponentConfig.GarbageCollectorController.ConcurrentGCSyncs)
    go garbageCollector.Run(workers, ctx.Stop)

    // 5、监听集群中的 DeletableResources
    go garbageCollector.Sync(gcClientset.Discovery(), 30*time.Second, ctx.Stop)

    // 6、注册 debug 接口
    return garbagecollector.NewDebugHandler(garbageCollector), true, nil
}

func NewGarbageCollector(......) (*GarbageCollector, error) {
    ......
    gc := &GarbageCollector{
        ......
    }
    gb := &GraphBuilder{
        ......
    }
    if err := gb.syncMonitors(deletableResources); err != nil {
        utilruntime.HandleError(fmt.Errorf("failed to sync all monitors: %v", err))
    }
    gc.dependencyGraphBuilder = gb

    return gc, nil
}

func (gb *GraphBuilder) syncMonitors(resources map[schema.GroupVersionResource]struct{}) error {
    gb.monitorLock.Lock()
    defer gb.monitorLock.Unlock()

    ......
    for resource := range resources {
        if _, ok := gb.ignoredResources[resource.GroupResource()]; ok {
            continue
        }
        ......
        kind, err := gb.restMapper.KindFor(resource)
        if err != nil {
            errs = append(errs, fmt.Errorf("couldn't look up resource %q: %v", resource, err))
            continue
        }
        // 为 resource 的 controller 注册 eventHandler 
        c, s, err := gb.controllerFor(resource, kind)
        if err != nil {
            errs = append(errs, fmt.Errorf("couldn't start monitor for resource %q: %v", resource, err))
            continue
        }
        current[resource] = &monitor{store: s, controller: c}
        added++
    }
    gb.monitors = current

    for _, monitor := range toRemove {
        if monitor.stopCh != nil {
            close(monitor.stopCh)
        }
    }
    return utilerrors.NewAggregate(errs)
}

func (gb *GraphBuilder) controllerFor(resource schema.GroupVersionResource, kind schema.GroupVersionKind) (cache.Controller, cache.Store, error) {
    handlers := cache.ResourceEventHandlerFuncs{
        AddFunc: func(obj interface{}) {
            event := &event{
                eventType: addEvent,
                obj:       obj,
                gvk:       kind,
            }
            // 将对应的 event push 到 graphChanges 队列中
            gb.graphChanges.Add(event)
        },
        UpdateFunc: func(oldObj, newObj interface{}) {
            event := &event{
                eventType: updateEvent,
                obj:       newObj,
                oldObj:    oldObj,
                gvk:       kind,
            }
            // 将对应的 event push 到 graphChanges 队列中
            gb.graphChanges.Add(event)
        },
        DeleteFunc: func(obj interface{}) {
            if deletedFinalStateUnknown, ok := obj.(cache.DeletedFinalStateUnknown); ok {
                obj = deletedFinalStateUnknown.Obj
            }
            event := &event{
                eventType: deleteEvent,
                obj:       obj,
                gvk:       kind,
            }
            // 将对应的 event push 到 graphChanges 队列中
            gb.graphChanges.Add(event)
        },
    }
    shared, err := gb.sharedInformers.ForResource(resource)
    if err != nil {
        return nil, nil, err
    }
    shared.Informer().AddEventHandlerWithResyncPeriod(handlers, ResourceResyncTime)
    return shared.Informer().GetController(), shared.Informer().GetStore(), nil
}

                     |--> ctx.ClientBuilder.
                     |    ClientOrDie
                     |
                     |
                     |--> cacheddiscovery.
                     |    NewMemCacheClient
                     |                                                                  |--> gb.sharedInformers.
                     |                                                                  |       ForResource
                     |                                                                  |
startGarbage     ----|--> garbagecollector.  --> gb.syncMonitors --> gb.controllerFor --|
CollectorController  |    NewGarbageCollector                                           |
                     |                                                                  |
                     |                                                                  |--> shared.Informer().
                     |                                                                    AddEventHandlerWithResyncPeriod
                     |--> garbageCollector.Run
                     |
                     |
                     |--> garbageCollector.Sync
                     |
                     |
                     |--> garbagecollector.NewDebugHandler

func (gc *GarbageCollector) Run(workers int, stopCh <-chan struct{}) {
    defer utilruntime.HandleCrash()
    defer gc.attemptToDelete.ShutDown()
    defer gc.attemptToOrphan.ShutDown()
    defer gc.dependencyGraphBuilder.graphChanges.ShutDown()

    defer klog.Infof("Shutting down garbage collector controller")

    // 1、调用 gc.dependencyGraphBuilder.Run 启动所有的 monitors 即 informers，并且启动一个 goroutine 处理 graphChanges 中的事件将其分别放到 GraphBuilder 的 attemptToDelete 和 attemptToOrphan 两个 队列中；
    go gc.dependencyGraphBuilder.Run(stopCh)

    // 2、等待 informers 的 cache 同步完成
    if !cache.WaitForNamedCacheSync("garbage collector", stopCh, gc.dependencyGraphBuilder.IsSynced) {
        return
    }

    for i := 0; i < workers; i++ {
        // 3、启动多个 goroutine 调用 gc.runAttemptToDeleteWorker 处理 attemptToDelete 中的事件
        go wait.Until(gc.runAttemptToDeleteWorker, 1*time.Second, stopCh)
        // 4、启动多个 goroutine 调用 gc.runAttemptToOrphanWorker 处理 attemptToDelete 中的事件
        go wait.Until(gc.runAttemptToOrphanWorker, 1*time.Second, stopCh)
    }

    <-stopCh
}

type GraphBuilder struct {
    restMapper meta.RESTMapper

    // informers
    monitors    monitors
    monitorLock sync.RWMutex

    // 当 kube-controller-manager 中所有的 controllers 都启动后，informersStarted 会被 close 掉
    // informersStarted 会被 close 掉的调用程序在 kube-controller-manager 的启动流程中
    informersStarted <-chan struct{}

    stopCh <-chan struct{}

    // 当调用 GraphBuilder 的 run 方法时，running 会被设置为 true
    running bool

    metadataClient metadata.Interface
    
    // informers 监听到的事件会放在 graphChanges 中
    graphChanges workqueue.RateLimitingInterface
    
    // 维护所有对象的依赖关系 
    uidToNode *concurrentUIDToNode
    
    // GarbageCollector 作为消费者要处理 attemptToDelete 和 attemptToOrphan 两个队列中的事件
    attemptToDelete workqueue.RateLimitingInterface
    attemptToOrphan workqueue.RateLimitingInterface

    absentOwnerCache *UIDCache
    sharedInformers  controller.InformerFactory
    // 不需要被 gc 的资源
    ignoredResources map[schema.GroupResource]struct{}
}

type concurrentUIDToNode struct {
    uidToNodeLock sync.RWMutex
    uidToNode     map[types.UID]*node
}

type node struct {
    identity objectReference
    
    dependentsLock sync.RWMutex
    // 其依赖项指 metadata.ownerReference 中的对象
    dependents map[*node]struct{}

    deletingDependents     bool
    deletingDependentsLock sync.RWMutex

    beingDeleted     bool
    beingDeletedLock sync.RWMutex
    
    // 当 virtual 值为 true 时，此时不确定该对象是否存在于 apiserver 中
    virtual     bool
    virtualLock sync.RWMutex
    
    // 对象本身的 OwnerReference 列表
    owners []metav1.OwnerReference
}

func (gb *GraphBuilder) Run(stopCh <-chan struct{}) {
    klog.Infof("GraphBuilder running")
    defer klog.Infof("GraphBuilder stopping")

    gb.monitorLock.Lock()
    gb.stopCh = stopCh
    gb.running = true
    gb.monitorLock.Unlock()

    gb.startMonitors()
    
    // 调用 gb.runProcessGraphChanges
    // 此处为死循环，除非收到 stopCh 信号，否则下面的代码不会被执行到
    wait.Until(gb.runProcessGraphChanges, 1*time.Second, stopCh)

    // 若执行到此处说明收到了 stopCh 的信号，此时需要停止所有的 running monitors
    gb.monitorLock.Lock()
    defer gb.monitorLock.Unlock()
    monitors := gb.monitors
    stopped := 0
    for _, monitor := range monitors {
        if monitor.stopCh != nil {
            stopped++
            close(monitor.stopCh)
        }
    }

    gb.monitors = nil
}

func (gb *GraphBuilder) startMonitors() {
    gb.monitorLock.Lock()
    defer gb.monitorLock.Unlock()
		
    // 1、当 GraphBuilder 调用 run 方法后，running 会设置为 true
    if !gb.running {
        return
    }

    // 2、当 kube-controller-manager 中所有的 controllers 在启动流程中都启动后
    //    会 close 掉 informersStarted
    <-gb.informersStarted

    // 3、启动所有 informer
    monitors := gb.monitors
    started := 0
    for _, monitor := range monitors {
        if monitor.stopCh == nil {
            monitor.stopCh = make(chan struct{})
            gb.sharedInformers.Start(gb.stopCh)
            go monitor.Run()
            started++
        }
    }
}

func (gb *GraphBuilder) runProcessGraphChanges() {
    for gb.processGraphChanges() {
    }
}

func (gb *GraphBuilder) processGraphChanges() bool {
    // 1、从 graphChanges 取出一个 event
    item, quit := gb.graphChanges.Get()
    if quit {
        return false
    }
    defer gb.graphChanges.Done(item)
    event, ok := item.(*event)
    if !ok {
        utilruntime.HandleError(fmt.Errorf("expect a *event, got %v", item))
        return true
    }
    obj := event.obj
    accessor, err := meta.Accessor(obj)
    if err != nil {
        utilruntime.HandleError(fmt.Errorf("cannot access obj: %v", err))
        return true
    }

    // 2、若存在 node 对象，从 uidToNode 中取出该 event 的 node 对象
    existingNode, found := gb.uidToNode.Read(accessor.GetUID())
    if found {
        existingNode.markObserved()
    }
    switch {
    // 3、若 event 为 add 或 update 类型以及对应的 node 对象不存在时
    case (event.eventType == addEvent || event.eventType == updateEvent) && !found:
        // 4、为 node 创建 event 对象
        newNode := &node{
            ......
        }
        // 5、在 uidToNode 中添加该 node 对象
        gb.insertNode(newNode)

        // 6、检查并处理 node 的删除操作 
        gb.processTransitions(event.oldObj, accessor, newNode)
        
    // 7、若 event 为 add 或 update 类型以及对应的 node 对象存在时
    case (event.eventType == addEvent || event.eventType == updateEvent) && found:
        added, removed, changed := referencesDiffs(existingNode.owners, accessor.GetOwnerReferences())
        // 8、若 node 的 owners 有变化
        if len(added) != 0 || len(removed) != 0 || len(changed) != 0 {

            gb.addUnblockedOwnersToDeleteQueue(removed, changed)
            // 9、更新 uidToNode 中的 owners
            existingNode.owners = accessor.GetOwnerReferences()
            // 10、添加更新后 Owners 对应的 dependent
            gb.addDependentToOwners(existingNode, added)
            // 11、移除旧 owners 对应的 dependents
            gb.removeDependentFromOwners(existingNode, removed)
        }
				
        // 12、检查是否处于删除状态
        if beingDeleted(accessor) {
            existingNode.markBeingDeleted()
        }
        // 13、检查并处理 node 的删除操作 
        gb.processTransitions(event.oldObj, accessor, existingNode)
        
    // 14、若为 delete event
    case event.eventType == deleteEvent:
        if !found {
            return true
        }
        // 15、从 uidToNode 中删除该 node
        gb.removeNode(existingNode)
        existingNode.dependentsLock.RLock()
        defer existingNode.dependentsLock.RUnlock()
        if len(existingNode.dependents) > 0 {
            gb.absentOwnerCache.Add(accessor.GetUID())
        }
        // 16、删除该 node 的 dependents
        for dep := range existingNode.dependents {
            gb.attemptToDelete.Add(dep)
        }
        // 17、删除该 node 处于删除阻塞状态的 owner 
        for _, owner := range existingNode.owners {
            ownerNode, found := gb.uidToNode.Read(owner.UID)
            if !found || !ownerNode.isDeletingDependents() {
                continue
            }
            gb.attemptToDelete.Add(ownerNode)
        }
    }
    return true
}

func (gb *GraphBuilder) processTransitions(oldObj interface{}, newAccessor metav1.Object, n *node) {
    if startsWaitingForDependentsOrphaned(oldObj, newAccessor) {
        gb.attemptToOrphan.Add(n)
        return
    }
    if startsWaitingForDependentsDeleted(oldObj, newAccessor) {
        n.markDeletingDependents()
        for dep := range n.dependents {
            gb.attemptToDelete.Add(dep)
        }
        gb.attemptToDelete.Add(n)
    }
}

func (gc *GarbageCollector) runAttemptToDeleteWorker() {
    for gc.attemptToDeleteWorker() {
    }
}

func (gc *GarbageCollector) attemptToDeleteWorker() bool {
    item, quit := gc.attemptToDelete.Get()
    gc.workerLock.RLock()
    defer gc.workerLock.RUnlock()
    if quit {
        return false
    }
    defer gc.attemptToDelete.Done(item)
    n, ok := item.(*node)
    if !ok {
        utilruntime.HandleError(fmt.Errorf("expect *node, got %#v", item))
        return true
    }
    err := gc.attemptToDeleteItem(n)
    if err != nil {
        if _, ok := err.(*restMappingError); ok {
            klog.V(5).Infof("error syncing item %s: %v", n, err)
        } else {
            utilruntime.HandleError(fmt.Errorf("error syncing item %s: %v", n, err))
        }
        gc.attemptToDelete.AddRateLimited(item)
    } else if !n.isObserved() {
        gc.attemptToDelete.AddRateLimited(item)
    }
    return true
}

func (gc *GarbageCollector) attemptToDeleteItem(item *node) error {
    // 1、判断 node 是否处于删除状态
    if item.isBeingDeleted() && !item.isDeletingDependents() {
        return nil
    }
		
    // 2、从 apiserver 获取该 node 最新的状态
    latest, err := gc.getObject(item.identity)
    switch {
    case errors.IsNotFound(err):
        gc.dependencyGraphBuilder.enqueueVirtualDeleteEvent(item.identity)
        item.markObserved()
        return nil
    case err != nil:
        return err
    }
    
    // 3、判断该 node 最新状态的 uid 是否等于本地缓存中的 uid
    if latest.GetUID() != item.identity.UID {
        gc.dependencyGraphBuilder.enqueueVirtualDeleteEvent(item.identity)
        item.markObserved()
        return nil
    }

    // 4、判断该 node 当前是否处于删除 dependents 状态中
    if item.isDeletingDependents() {
        return gc.processDeletingDependentsItem(item)
    }

    // 5、检查 node 是否还存在 ownerReferences
    ownerReferences := latest.GetOwnerReferences()
    if len(ownerReferences) == 0 {
        return nil
    }

    // 6、对 ownerReferences 进行分类
    solid, dangling, waitingForDependentsDeletion, err := gc.classifyReferences(item, ownerReferences)
    if err != nil {
        return err
    }
    switch {
    // 7、存在不处于删除状态的 owner
    case len(solid) != 0:
        if len(dangling) == 0 && len(waitingForDependentsDeletion) == 0 {
            return nil
        }
        ownerUIDs := append(ownerRefsToUIDs(dangling), ownerRefsToUIDs(waitingForDependentsDeletion)...)
        patch := deleteOwnerRefStrategicMergePatch(item.identity.UID, ownerUIDs...)
        _, err = gc.patch(item, patch, func(n *node) ([]byte, error) {
            return gc.deleteOwnerRefJSONMergePatch(n, ownerUIDs...)
        })
        return err
    // 8、node 的 owner 处于 waitingForDependentsDeletion 状态并且 node 
    //   的 dependents 未被完全删除
    case len(waitingForDependentsDeletion) != 0 && item.dependentsLength() != 0:
        deps := item.getDependents()
        // 9、删除 dependents
        for _, dep := range deps {
            if dep.isDeletingDependents() {
                patch, err := item.unblockOwnerReferencesStrategicMergePatch()
                if err != nil {
                    return err
                }
                if _, err := gc.patch(item, patch, gc.unblockOwnerReferencesJSONMergePatch); err != nil {
                    return err
                }
                break
            }
        }
        // 10、以 Foreground 模式删除 node 对象
        policy := metav1.DeletePropagationForeground
        return gc.deleteObject(item.identity, &policy)
    // 11、该 node 已经没有任何依赖了，按照 node 中声明的删除策略调用 apiserver 的接口删除
    default:
        var policy metav1.DeletionPropagation
        switch {
        case hasOrphanFinalizer(latest):
            policy = metav1.DeletePropagationOrphan
        case hasDeleteDependentsFinalizer(latest):
            policy = metav1.DeletePropagationForeground
        default:
            policy = metav1.DeletePropagationBackground
        }
        return gc.deleteObject(item.identity, &policy)
    }
}

func (gc *GarbageCollector) runAttemptToOrphanWorker() {
    for gc.attemptToOrphanWorker() {
    }
}

func (gc *GarbageCollector) attemptToOrphanWorker() bool {
    item, quit := gc.attemptToOrphan.Get()
    gc.workerLock.RLock()
    defer gc.workerLock.RUnlock()
    if quit {
        return false
    }
    defer gc.attemptToOrphan.Done(item)
    owner, ok := item.(*node)
    if !ok {
        return true
    }
    owner.dependentsLock.RLock()
    dependents := make([]*node, 0, len(owner.dependents))
    for dependent := range owner.dependents {
        dependents = append(dependents, dependent)
    }
    owner.dependentsLock.RUnlock()
    err := gc.orphanDependents(owner.identity, dependents)
    if err != nil {
        gc.attemptToOrphan.AddRateLimited(item)
        return true
    }
    // 更新 owner, 从 finalizers 列表中移除 "orphaningFinalizer"
    err = gc.removeFinalizer(owner, metav1.FinalizerOrphanDependents)
    if err != nil {
        gc.attemptToOrphan.AddRateLimited(item)
    }
    return true
}

func (gc *GarbageCollector) Sync(discoveryClient discovery.ServerResourcesInterface, period time.Duration, stopCh <-chan struct{}) {
    oldResources := make(map[schema.GroupVersionResource]struct{})
    wait.Until(func() {
        // 1、获取集群内所有的 DeletableResources 作为 newResources
        newResources := GetDeletableResources(discoveryClient)

        if len(newResources) == 0 {
            return
        }

        // 2、判断集群中的资源是否有变化
        if reflect.DeepEqual(oldResources, newResources) {
            return
        }

        gc.workerLock.Lock()
        defer gc.workerLock.Unlock()
       
        // 3、开始更新 GraphBuilder 中的 monitors      
        attempt := 0
        wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) {
            attempt++

            if attempt > 1 {
                newResources = GetDeletableResources(discoveryClient)
                if len(newResources) == 0 {
                    return false, nil
                }
            }

            gc.restMapper.Reset()
            // 4、调用 gc.resyncMonitors 同步 newResources
            if err := gc.resyncMonitors(newResources); err != nil {
                return false, nil
            }

            // 5、等待所有 monitors 的 cache 同步完成
            if !cache.WaitForNamedCacheSync("garbage collector", waitForStopOrTimeout(stopCh, period), gc.dependencyGraphBuilder.IsSynced) {
                return false, nil
            }

            return true, nil
        }, stopCh)

        // 6、更新 oldResources
        oldResources = newResources
    }, period, stopCh)
}

func GetDeletableResources(discoveryClient discovery.ServerResourcesInterface) map[schema.GroupVersionResource]struct{} {
    // 1、调用 discoveryClient.ServerPreferredResources 方法获取集群内所有的 resource 信息
    preferredResources, err := discoveryClient.ServerPreferredResources()
    if err != nil {
        if discovery.IsGroupDiscoveryFailedError(err) {		
        	......
        } else {
        	......
        }
    }
    if preferredResources == nil {
        return map[schema.GroupVersionResource]struct{}{}
    }
    // 2、调用 discovery.FilteredBy 过滤出 deletableResources
    deletableResources := discovery.FilteredBy(discovery.SupportsAllVerbs{Verbs: []string{"delete", "list", "watch"}}, preferredResources)
    deletableGroupVersionResources := map[schema.GroupVersionResource]struct{}{}
    for _, rl := range deletableResources {
        gv, err := schema.ParseGroupVersion(rl.GroupVersion)
        if err != nil {
            continue
        }
        for i := range rl.APIResources {
            deletableGroupVersionResources[schema.GroupVersionResource{Group: gv.Group, Version: gv.Version, Resource: rl.APIResources[i].Name}] =       struct{}{}
        }
    }

    return deletableGroupVersionResources
}

func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) {
    // 1、获取集群内所有的 GroupList
    serverGroupList, err := d.ServerGroups()
    if err != nil {
        return nil, err
    }

    // 2、通过 serverGroupList 获取到对应的 resource
    groupVersionResources, failedGroups := fetchGroupVersionResources(d, serverGroupList)

    result := []*metav1.APIResourceList{}
    grVersions := map[schema.GroupResource]string{}                         // selected version of a GroupResource
    grAPIResources := map[schema.GroupResource]*metav1.APIResource{}        // selected APIResource for a GroupResource
    gvAPIResourceLists := map[schema.GroupVersion]*metav1.APIResourceList{} // blueprint for a APIResourceList for later grouping

    // 3、格式化 resource 
    for _, apiGroup := range serverGroupList.Groups {
        for _, version := range apiGroup.Versions {
            groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}

            apiResourceList, ok := groupVersionResources[groupVersion]
            if !ok {
                continue
            }

            emptyAPIResourceList := metav1.APIResourceList{
                GroupVersion: version.GroupVersion,
            }
            gvAPIResourceLists[groupVersion] = &emptyAPIResourceList
            result = append(result, &emptyAPIResourceList)
            
            for i := range apiResourceList.APIResources {
                apiResource := &apiResourceList.APIResources[i]
                if strings.Contains(apiResource.Name, "/") {
                    continue
                }
                gv := schema.GroupResource{Group: apiGroup.Name, Resource: apiResource.Name}
                if _, ok := grAPIResources[gv]; ok && version.Version != apiGroup.PreferredVersion.Version {
                    continue
                }
                grVersions[gv] = version.Version
                grAPIResources[gv] = apiResource
            }
        }
    }

    for groupResource, apiResource := range grAPIResources {
        version := grVersions[groupResource]
        groupVersion := schema.GroupVersion{Group: groupResource.Group, Version: version}
        apiResourceList := gvAPIResourceLists[groupVersion]
        apiResourceList.APIResources = append(apiResourceList.APIResources, *apiResource)
    }

    if len(failedGroups) == 0 {
        return result, nil
    }

    return result, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}

                                                      |--> d.ServerGroups
                                                      |
                        |--> discoveryClient.       --|
                        |  ServerPreferredResources   |
                        |                             |--> fetchGroupVersionResources
GetDeletableResources --|
                        |
                        |--> discovery.FilteredBy

func (gc *GarbageCollector) resyncMonitors(deletableResources map[schema.                          GroupVersionResource]struct{}) error {
    if err := gc.dependencyGraphBuilder.syncMonitors(deletableResources); err != nil {
        return err
    }
    gc.dependencyGraphBuilder.startMonitors()
    return nil
}

func startGarbageCollectorController(ctx ControllerContext) (http.Handler, bool, error) {
	......
	return garbagecollector.NewDebugHandler(garbageCollector), true, nil
}

$ curl http://192.168.99.108:10252/debug/controllers/garbagecollector/graph  > tmp.dot

$ curl http://192.168.99.108:10252/debug/controllers/garbagecollector/graph\?uid=f9555d53-2b5f-4702-9717-54a313ed4fe8 > tmp.dot

// 生成 svg 文件
$ dot -Tsvg -o graph.svg tmp.dot

// 然后在浏览器中打开 svg 文件

$ kubectl get deployment  nginx-deployment
NAME               READY   UP-TO-DATE   AVAILABLE   AGE
nginx-deployment   2/2     2            2           43s

$ kubectl get rs nginx-deployment-69b6b4c5cd
NAME                          DESIRED   CURRENT   READY   AGE
nginx-deployment-69b6b4c5cd   2         2         2       57s

$ kubectl get pod
NAME                                READY   STATUS    RESTARTS   AGE
nginx-deployment-69b6b4c5cd-26dsn   1/1     Running   0          66s
nginx-deployment-69b6b4c5cd-6rqqc   1/1     Running   0          64s

$ kubectl edit rs nginx-deployment-69b6b4c5cd   
  
// deployment 关联的 rs 对象  
apiVersion: apps/v1
kind: ReplicaSet
metadata:
  name: nginx-deployment-69b6b4c5cd
  namespace: default
  ownerReferences:
  - apiVersion: apps/v1
    blockOwnerDeletion: true
    controller: true
    kind: Deployment
    name: nginx-deployment
    uid: 40a1044e-03d1-48bc-8806-cb79d781c946
  finalizers:
  - foregroundDeletion       // 为 rs 手动添加的 Foreground 策略
  ......
spec:
  replicas: 2
  ......
status:
  ......

$ systemctl stop kubelet

// node 处于 NotReady 状态
$ kubectl get node
NAME       STATUS     ROLES    AGE     VERSION
minikube   NotReady   master   6d11h   v1.16.2

// 以 Foreground 策略删除 deployment
$ curl -k -v -XDELETE  -H "Accept: application/json" -H "Content-Type: application/json" -d '{"propagationPolicy":"Foreground"}' 'https://192.168.99.108:8443/apis/apps/v1/namespaces/default/deployments/nginx-deployment'

                    monitors (producer)
                          |
                          |
                          ∨
                  graphChanges queue
                          |
                          |
                          ∨
                  processGraphChanges
                          |
                          |
                          ∨
          -------------------------------
          |                             |
          |                             |
          ∨                             ∨
attemptToDelete queue         attemptToOrphan queue
          |                             |
          |                             |
          ∨                             ∨
  AttemptToDeleteWorker       AttemptToOrphanWorker
      (consumer)                    (consumer)