Fix issues spotted by Coverity from fmap recursion fix

CID 361074: fmap.c: Possible invalid dereference if status != success
and the new map was not yet allocated.

CID 361077: others.c: Structurally dead code revealed a bug in the
cli_recursion_stack_get_size() function.

CID 361080, 361078, 361083: sigtool.c: Inverted check for if engine
needs to be free'd, could leak the engine structure.

CID 361075: sigtool.c: Missed a `return -1` that should've been `goto
done;` and would leak the new_map buffer.

CID 361079: sigtool/vba.c: Checking if we should free the new_map on
failure only if ctx also needs to be free'd, which would leak the
new_map if ctx was not allocated yet.

libclamav/fmap.c

@@ -865,11 +865,13 @@ fmap_t *fmap_open_memory(const void *start, size_t len, const char *name)
 
 done:
     if (CL_SUCCESS != status) {
-        if (NULL != m->name) {
-            free(m->name);
+        if (NULL != m) {
+            if (NULL != m->name) {
+                free(m->name);
+            }
+            free(m);
+            m = NULL;
         }
-        free(m);
-        m = NULL;
     }
 
     return m;

libclamav/others.c

@@ -1532,7 +1532,7 @@ size_t cli_recursion_stack_get_size(cli_ctx *ctx, int index)
         // Layer too low, does not exist.
         // Most likely we're at the top layer and there is no container. That's okay.
         return ctx->recursion_stack[0].size;
-    } else {
+    } else if (ctx->recursion_level < (uint32_t)index_ignoring_normalized_layers) {
         // layer too high, does not exist. This should never happen!
         return 0;
     }

sigtool/sigtool.c

@@ -288,7 +288,7 @@ done:
     if (NULL != ctx.recursion_stack) {
         free(ctx.recursion_stack);
     }
-    if (NULL == engine) {
+    if (NULL != engine) {
         cl_engine_free(engine);
     }
     if (-1 != fd) {
@@ -2297,7 +2297,7 @@ done:
     if (NULL != ctx.recursion_stack) {
         free(ctx.recursion_stack);
     }
-    if (NULL == engine) {
+    if (NULL != engine) {
         cl_engine_free(engine);
     }
 }
@@ -3437,7 +3437,7 @@ static int dumpcerts(const struct optstruct *opts)
     /* build engine */
     if (!(engine = cl_engine_new())) {
         mprintf("!dumpcerts: Can't create new engine\n");
-        return -1;
+        goto done;
     }
     cl_engine_set_num(engine, CL_ENGINE_AC_ONLY, 1);
 
@@ -3510,7 +3510,7 @@ done:
     if (NULL != ctx.recursion_stack) {
         free(ctx.recursion_stack);
     }
-    if (NULL == engine) {
+    if (NULL != engine) {
         cl_engine_free(engine);
     }
     if (-1 != fd) {

sigtool/vba.c

@@ -130,10 +130,10 @@ cli_ctx *convenience_ctx(int fd)
 
 done:
     if (CL_SUCCESS != status) {
+        if (NULL != new_map) {
+            funmap(new_map);
+        }
         if (NULL != ctx) {
-            if (NULL != new_map) {
-                funmap(new_map);
-            }
             if (NULL != ctx->options) {
                 free(ctx->options);
             }