From 2f9d26427ce5f49ae624ebe8f4dedfe3a436e83f Mon Sep 17 00:00:00 2001 From: "Alexander \"Arav\" Andreev" Date: Mon, 3 May 2021 02:33:14 +0400 Subject: [PATCH] Now incrementing _files_downloaded happens when _progress_callback is set. And made super() with no args. --- scrapthechan/scrapers/threadedscraper.py | 35 ++++++++++++------------ 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/scrapthechan/scrapers/threadedscraper.py b/scrapthechan/scrapers/threadedscraper.py index f745649..dbccd83 100644 --- a/scrapthechan/scrapers/threadedscraper.py +++ b/scrapthechan/scrapers/threadedscraper.py @@ -7,25 +7,26 @@ from multiprocessing.pool import ThreadPool from scrapthechan.scraper import Scraper from scrapthechan.fileinfo import FileInfo + __all__ = ["ThreadedScraper"] + class ThreadedScraper(Scraper): - def __init__(self, save_directory: str, files: List[FileInfo], - download_progress_callback: Callable[[int], None] = None) -> None: - super(ThreadedScraper, self).__init__(save_directory, files, - download_progress_callback) - self._files_downloaded = 0 - self._files_downloaded_mutex = Lock() + def __init__(self, save_directory: str, files: List[FileInfo], + download_progress_callback: Callable[[int], None] = None) -> None: + super().__init__(save_directory, files, download_progress_callback) + self._files_downloaded = 0 + self._files_downloaded_mutex = Lock() - def run(self): - pool = ThreadPool(cpu_count() * 2) - pool.map(self._thread_run, self._files) - pool.close() - pool.join() + def run(self): + pool = ThreadPool(cpu_count() * 2) + pool.map(self._thread_run, self._files) + pool.close() + pool.join() - def _thread_run(self, f: FileInfo): - with self._files_downloaded_mutex: - self._files_downloaded += 1 - if not self._progress_callback is None: - self._progress_callback(self._files_downloaded) - self._download_file(f) + def _thread_run(self, f: FileInfo): + if not self._progress_callback is None: + with self._files_downloaded_mutex: + self._files_downloaded += 1 + self._progress_callback(self._files_downloaded) + self._download_file(f)