Beginning of redis support

config_manager.py

@@ -19,6 +19,7 @@ class SectionConfig:
         '_log_backups': 7,
         '_log_level': 20,
         '_unit_test': True,
+        '_enable_redis': True
     }
 
     def append(self, key, item):
@@ -92,7 +93,10 @@ class ConfigManager:
 
         self.__config.read(params_file if params_file is not None else 'params/params.ini')
 
-        self.__init_logger(self.__config['app'].getint('_log_level'), self.__config['app'].getint('_log_backups'))
+        self.log_level = self.__config['app'].getint('_log_level')
+        self.log_backups = self.__config['app'].getint('_log_backups')
+
+        self.init_logger()
 
         logging.info(f"Container build date : {os.environ.get('DATE')}, git revision : {os.environ.get('GIT_BRANCH')} - {os.environ.get('GIT_REVISION')}")
 
@@ -100,12 +104,11 @@ class ConfigManager:
         self.__load_metadata()
         self.__set_config_objects()
 
-    @staticmethod
-    def __init_logger(level=30, backup_count=7):
-        logging.basicConfig(level=level, format='[%(asctime)s][%(name)s][%(levelname)s] %(message)s', datefmt='%d-%m-%y %H:%M:%S')
+    def init_logger(self):
+        logging.basicConfig(level=self.log_level, format='[%(asctime)s][%(name)s][%(levelname)s] %(message)s', datefmt='%d-%m-%y %H:%M:%S')
 
-        time_handler = handlers.TimedRotatingFileHandler('logs/ydl_api_ng', when='midnight', interval=1, backupCount=backup_count)
-        time_handler.setLevel(level)
+        time_handler = handlers.TimedRotatingFileHandler('logs/ydl_api_ng', when='midnight', interval=1, backupCount=self.log_backups)
+        time_handler.setLevel(self.log_level)
         time_handler.setFormatter(logging.Formatter('[%(asctime)s][%(name)s][%(levelname)s] %(message)s', datefmt='%d-%m-%y %H:%M:%S'))
         logging.getLogger().addHandler(time_handler)
 

download_manager.py

@@ -4,6 +4,8 @@ import logging
 from urllib.parse import urlparse
 
 import yt_dlp as ydl
+from redis import Redis
+from rq import Queue
 
 import config_manager
 from params import progress_hooks, postprocessor_hooks, ydl_api_hooks
@@ -43,6 +45,8 @@ class DownloadManager:
         is_video = self.check_if_video()
         self.is_video = is_video if is_video is not None else False
 
+        self.enable_redis = self.__cm.get_app_params().get('_enable_redis')
+
         if post_body is not None and post_body.get('presets') is not None and len(post_body.get('presets')) > 0:
             self.get_presets_from_post_request(post_body.get('presets'))
         else:
@@ -259,23 +263,32 @@ class DownloadManager:
 
         ydl_opts = copy.deepcopy(preset)
 
-        ydl_opts.append('progress_hooks', [functools.partial(progress_hooks.handler, ydl_opts, self, self.__cm), functools.partial(self.progress_hooks_proxy)])
-        ydl_opts.append('postprocessor_hooks', [functools.partial(postprocessor_hooks.handler, ydl_opts, self, self.__cm)])
+        ydl_opts.append('progress_hooks', [functools.partial(progress_hooks.handler, ydl_opts, self, self.get_current_config_manager()), functools.partial(self.progress_hooks_proxy)])
+        ydl_opts.append('postprocessor_hooks', [functools.partial(postprocessor_hooks.handler, ydl_opts, self, self.get_current_config_manager())])
         ydl_opts.append('logger', logging.getLogger('youtube-dlp'))
 
-        ydl_api_hooks.pre_download_handler(ydl_opts, self, self.__cm)
+        ydl_api_hooks.pre_download_handler(ydl_opts, self, self.get_current_config_manager())
+
+        if self.enable_redis:
+            queue = Queue('ydl_api_ng', connection=Redis())
+            redis_id = queue.enqueue(self.send_download_order, args=[ydl_opts, True], job_timeout=-1).id
+            preset.append('_redis_id', redis_id)
+        else:
+            preset.append('_redis_id', None)
+            self.send_download_order(ydl_opts)
+
+    def send_download_order(self, ydl_opts):
+        if self.enable_redis:
+            self.get_current_config_manager().init_logger()
 
         try:
             with ydl.YoutubeDL(ydl_opts.get_all()) as dl:
-                download_result = dl.download([self.url]) == 0
+                dl.download([self.url])
                 ydl_opts.append('__download_exception_message', None)
         except ydl.utils.DownloadError as error:
-            download_result = False
             ydl_opts.append('__download_exception_message', str(error))
 
-        ydl_api_hooks.post_download_handler(ydl_opts, self, self.__cm, self.downloaded_files)
-
-        return download_result
+        ydl_api_hooks.post_download_handler(ydl_opts, self, self.get_current_config_manager(), self.downloaded_files)
 
     def process_downloads(self):
         for preset in self.presets:
@@ -343,3 +356,6 @@ class DownloadManager:
             'downloads_cannot_be_checked': self.downloads_cannot_be_checked,
             'downloads': presets_display,
         }
+
+    def get_current_config_manager(self):
+        return self.__cm

main.py

@@ -68,6 +68,9 @@ async def download_request(response: Response, background_tasks: BackgroundTasks
     response.status_code = dm.get_api_status_code()
 
     if response.status_code != 400:
+        if __cm.get_app_params().get('_enable_redis'):
+            dm.process_downloads()
+        else:
             background_tasks.add_task(dm.process_downloads)
 
     return dm.get_api_return_object()

params/params.ini

@@ -8,6 +8,8 @@ _log_level = 20
 _log_backups = 7
 _listen_port = 80
 _listen_ip = 0.0.0.0
+_enable_redis = true
+
 _allow_dangerous_post_requests = false
 
 ;;;

params/params.sample.ini

@@ -39,6 +39,7 @@ _log_level = 20
 _log_backups = 7
 _listen_port = 80
 _listen_ip = 0.0.0.0
+_enable_redis = true
 
 ;;; just for unit tests purpose, please don't mind
 _int_test = 42

params/params_metadata.ini

@@ -32,6 +32,7 @@ _float = _float_test
 _bool = _allow_dangerous_post_requests
         _bool_test
         _dev_mode
+        _enable_redis
         _enable_users_management
         allow_multiple_audio_streams
         allow_multiple_video_streams

unit_tests.py

@@ -16,6 +16,7 @@ class TestActualParametersFile(unittest.TestCase):
         self.assertFalse(self.config_manager.get_app_params().get('_enable_users_management'))
         self.assertEqual('0.0.0.0', self.config_manager.get_app_params().get('_listen_ip'))
         self.assertFalse(self.config_manager.get_app_params().get('_allow_dangerous_post_requests'))
+        self.assertTrue(self.config_manager.get_app_params().get('_enable_redis'))
 
 
 class TestConfig(unittest.TestCase):