Fix reload of ZooKeeper service discovery config (#2669)

Rational:

* When the config is reloaded and the provider context is canceled, we need to
  exit the current ZK `TargetProvider.Run` method as a new provider will be
  instantiated.
* In case `Stop` is called on the `ZookeeperTreeCache`, the update/events
  channel may not be closed as it is shared by multiple caches and would
  thus be double closed.
* Stopping all `zookeeperTreeCacheNode`s on teardown ensures all associated
  watcher go-routines will be closed eagerly rather than implicityly on
  connection close events.

discovery/zookeeper/zookeeper.go

@@ -93,6 +93,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
 	for {
 		select {
 		case <-ctx.Done():
+			return
 		case event := <-d.updates:
 			tg := &config.TargetGroup{
 				Source: event.Path,

util/treecache/treecache.go

@@ -168,7 +168,7 @@ func (tc *ZookeeperTreeCache) loop(path string) {
 				failureMode = false
 			}
 		case <-tc.stop:
-			close(tc.events)
+			tc.recursiveStop(tc.head)
 			return
 		}
 	}
@@ -264,3 +264,13 @@ func (tc *ZookeeperTreeCache) recursiveDelete(path string, node *zookeeperTreeCa
 		tc.recursiveDelete(path+"/"+name, childNode)
 	}
 }
+
+func (tc *ZookeeperTreeCache) recursiveStop(node *zookeeperTreeCacheNode) {
+	if !node.stopped {
+		node.done <- struct{}{}
+		node.stopped = true
+	}
+	for _, childNode := range node.children {
+		tc.recursiveStop(childNode)
+	}
+}