libhb: fix some initialized memory reads and type mismatches issues found by clang static analyzer.

libhb/decavcodec.c

@@ -2371,9 +2371,13 @@ static void compute_frame_duration( hb_work_private_t *pv )
             {
                 tb = &(st->r_frame_rate);
             }
+
+            if (tb != NULL)
+            {
                 duration = (double)tb->den / (double)tb->num;
             }
         }
+    }
     else if (pv->context->framerate.num && pv->context->framerate.den)
     {
         duration = (double)pv->context->framerate.den / (double)pv->context->framerate.num;

libhb/denoise.c

@@ -83,7 +83,7 @@ static void hqdn3d_precalc_coef(int16_t *ct, int depth, double dist25)
 
     for (i = -(256<<LUT_BITS); i < 256<<LUT_BITS; i++)
     {
-        double f = ((i<<(9-LUT_BITS)) + (1<<(8-LUT_BITS)) - 1) / 512.0; // midpoint of the bin
+        double f = (i * (1 << (9-LUT_BITS)) + (1<<(8-LUT_BITS)) - 1) / 512.0; // midpoint of the bin
         simil = FFMAX(0, 1.0 - fabs(f) / 255.0);
         C = pow(simil, gamma) * 256.0 * f;
         ct[(256<<LUT_BITS)+i] = lrint(C);
@@ -174,7 +174,7 @@ static void hqdn3d_denoise_depth(uint8_t *frame_src, uint8_t *frame_dst,
     if (!frame_ant)
     {
         uint8_t *src = frame_src;
-        (*frame_ant_ptr) = frame_ant = malloc(w*h*sizeof(uint16_t));
+        (*frame_ant_ptr) = frame_ant = calloc(w * h, sizeof(uint16_t));
         for (y = 0; y < h; y++, frame_src += sstride, frame_ant += w)
         {
             for (x = 0; x < w; x++)

libhb/encavcodecaudio.c

@@ -21,8 +21,8 @@ struct hb_work_private_s
     int              samples_per_frame;
     unsigned long    max_output_bytes;
     unsigned long    input_samples;
-    uint8_t        * output_buf;
-    uint8_t        * input_buf;
+    float          * output_buf;
+    float          * input_buf;
     hb_list_t      * list;
 
     SwrContext     * swresample;
@@ -418,7 +418,7 @@ static void Encode(hb_work_object_t *w, hb_buffer_list_t *list)
     {
         int ret;
 
-        hb_list_getbytes(pv->list, pv->input_buf,
+        hb_list_getbytes(pv->list, (uint8_t *)pv->input_buf,
                          pv->input_samples * sizeof(float), &pts, &pos);
 
         // Prepare input frame
@@ -434,7 +434,7 @@ static void Encode(hb_work_object_t *w, hb_buffer_list_t *list)
                                               pv->context->sample_fmt, 1);
         avcodec_fill_audio_frame(&frame,
                                  pv->context->ch_layout.nb_channels, pv->context->sample_fmt,
-                                 pv->output_buf, out_size, 1);
+                                 (uint8_t *)pv->output_buf, out_size, 1);
         if (pv->swresample != NULL)
         {
             int out_samples;

libhb/encvorbis.c

@@ -31,7 +31,7 @@ hb_work_object_t hb_encvorbis =
 
 struct hb_work_private_s
 {
-    uint8_t   *buf;
+    float     *buf;
     hb_job_t  *job;
     hb_list_t *list;
 
@@ -229,7 +229,7 @@ static hb_buffer_t* Encode(hb_work_object_t *w)
     }
 
     /* Process more samples */
-    hb_list_getbytes(pv->list, pv->buf, pv->input_samples * sizeof(float),
+    hb_list_getbytes(pv->list, (uint8_t *)pv->buf, pv->input_samples * sizeof(float),
                      &pv->pts, NULL);
     buffer = vorbis_analysis_buffer(&pv->vd, OGGVORBIS_FRAME_SIZE);
     for (i = 0; i < OGGVORBIS_FRAME_SIZE; i++)

libhb/handbrake/ports.h

@@ -116,7 +116,7 @@ HB_DIR* hb_opendir(const char *path);
 int hb_closedir(HB_DIR *dir);
 void hb_rewinddir(HB_DIR *dir);
 struct dirent * hb_readdir(HB_DIR *dir);
-int hb_mkdir(char * name);
+int hb_mkdir(const char *name);
 int hb_stat(const char *path, hb_stat_t *sb);
 FILE * hb_fopen(const char *path, const char *mode);
 char * hb_strr_dir_sep(const char *path);

libhb/nal_units.c

@@ -501,6 +501,12 @@ hb_buffer_t * hb_isomp4_hevc_nal_bitstream_insert_payloads(const uint8_t *data,
         return NULL;
     }
 
+    if ((seis == NULL && sei_count > 0) ||
+        (nals == NULL && nal_count > 0))
+    {
+        return NULL;
+    }
+
     for (int i = 0; i < sei_count; i++)
     {
         size_t msg_size = get_sei_msg_bytes(seis[i].payload, seis[i].payload_size, seis[i].type);

libhb/ports.c

@@ -806,7 +806,7 @@ char * hb_strr_dir_sep(const char *path)
  * Wrapper to the real mkdir, needed only because it doesn't take a
  * second argument on Win32. Grrr.
  ***********************************************************************/
-int hb_mkdir(char * path)
+int hb_mkdir(const char * path)
 {
 #ifdef SYS_MINGW
     wchar_t path_utf16[MAX_PATH];