libhb: fix various clang static analyzer warnings

libhb/comb_detect.c

@@ -1157,8 +1157,8 @@ static int comb_detect_init(hb_filter_object_t *filter,
     pv->gamma_spatial_threshold6  = 6 * pv->gamma_spatial_threshold;
     pv->spatial_threshold_squared = pv->spatial_threshold * pv->spatial_threshold;
     pv->spatial_threshold6        = 6 * pv->spatial_threshold;
-    pv->comb32detect_min = 10 << (pv->depth - 8);
+    pv->comb32detect_min = pv->depth >= 8 ? 10 << (pv->depth - 8) : 10;
-    pv->comb32detect_max = 15 << (pv->depth - 8);
+    pv->comb32detect_max = pv->depth >= 8 ? 15 << (pv->depth - 8) : 15;
 
     pv->cpu_count = hb_get_cpu_count();
 

libhb/decavsub.c

@@ -132,7 +132,7 @@ hb_decavsub_context_t * decavsubInit( hb_work_object_t * w, hb_job_t * job )
     if (ctx->pkt == NULL)
     {
         hb_log("decavsubInit: av_packet_alloc failed");
-        return NULL;
+        goto fail;
     }
 
     // avcodec may create or change subtitle header

libhb/encavcodec.c

@@ -899,11 +899,12 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
         }
         else
         {
-            int    size;
+            long   size;
             char * log;
 
             pv->file = hb_fopen(filename, "rb");
-            if (!pv->file) {
+            if (!pv->file)
+            {
                 if (strerror_r(errno, reason, 79) != 0)
                     strcpy(reason, "unknown -- strerror_r() failed");
 
@@ -915,6 +916,17 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job )
             fseek( pv->file, 0, SEEK_END );
             size = ftell( pv->file );
             fseek( pv->file, 0, SEEK_SET );
+
+            if (size == -1)
+            {
+                hb_error( "encavcodecInit: Failed to read %s", filename);
+                free(filename);
+                ret = 1;
+                fclose( pv->file );
+                pv->file = NULL;
+                goto done;
+            }
+
             log = malloc( size + 1 );
             log[size] = '\0';
             if (size > 0 &&

libhb/hb.c

@@ -2337,12 +2337,16 @@ static void redirect_thread_func(void * _data)
 #endif
     setvbuf(stderr, NULL, _IONBF, 0);
 
-    FILE * log_f = fdopen(pfd[0], "rb");
+    FILE *log_f = fdopen(pfd[0], "rb");
 
-    char line_buffer[500];
+    if (log_f != NULL)
-    while(fgets(line_buffer, 500, log_f) != NULL)
     {
-        hb_log_callback(line_buffer);
+        char line_buffer[500];
+        while(fgets(line_buffer, 500, log_f) != NULL)
+        {
+            hb_log_callback(line_buffer);
+        }
+        fclose(log_f);
     }
 }
 

libhb/muxavformat.c

@@ -277,6 +277,12 @@ static int avformatInit( hb_mux_object_t * m )
                      hb_list_count( job->list_subtitle );
     m->tracks = calloc(max_tracks, sizeof(hb_mux_data_t*));
 
+    if (m->tracks == NULL)
+    {
+        hb_error("muxavformat: calloc failed");
+        goto error;
+    }
+
     AVDictionary * av_opts = NULL;
     switch (job->mux)
     {
@@ -1207,11 +1213,14 @@ static int avformatInit( hb_mux_object_t * m )
     return 0;
 
 error:
-    for (ii = 0; ii < m->ntracks; ii++)
+    if (m->tracks)
     {
-        if (m->tracks[ii]->oc != NULL)
+        for (ii = 0; ii < m->ntracks; ii++)
         {
-            avformat_free_context(m->tracks[ii]->oc);
+            if (m->tracks[ii] != NULL && m->tracks[ii]->oc != NULL)
+            {
+                avformat_free_context(m->tracks[ii]->oc);
+            }
         }
     }
     av_dict_free(&av_opts);

libhb/platform/macosx/encvt.c

@@ -400,8 +400,10 @@ static OSType hb_vt_encoder_pixel_format_xlat(int vcodec, int profile, int color
             {
                 case HB_VT_H265_PROFILE_MAIN_10:
                     pix_fmt = hb_vt_get_best_pix_fmt(vcodec, "main-10");
+                    break;
                 case HB_VT_H265_PROFILE_MAIN_422_10:
                     pix_fmt = hb_vt_get_best_pix_fmt(vcodec, "main422-10");
+                    break;
             }
             break;
         default:

libhb/ssautil.c

@@ -356,7 +356,7 @@ static char ** get_fields(char * line, int last)
     {
         result[ii] = get_field(&pos);
     }
-    result[ii] = strdup(pos);
+    result[ii] = pos != NULL ? strdup(pos) : NULL;
 
     return result;
 }