Arav/ScrapTheChan
1
0
Fork 0
Code Releases 8 Activity

Now incrementing _files_downloaded happens when _progress_callback is set. And made super() with no args.

Browse Source
This commit is contained in:
Alexander Andreev 2021-05-03 02:33:14 +04:00
parent e7cf2e7c4b
commit 2f9d26427c
Signed by: Arav
GPG Key ID: 610DF2574456329F
1 changed files with 18 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapthechan/scrapers/threadedscraper.py
View File

@ -7,13 +7,14 @@ from multiprocessing.pool import ThreadPool
from scrapthechan.scraper import Scraper from scrapthechan.scraper import Scraper
from scrapthechan.fileinfo import FileInfo from scrapthechan.fileinfo import FileInfo
__all__ = ["ThreadedScraper"] __all__ = ["ThreadedScraper"]
class ThreadedScraper(Scraper): class ThreadedScraper(Scraper):
def __init__(self, save_directory: str, files: List[FileInfo], def __init__(self, save_directory: str, files: List[FileInfo],
download_progress_callback: Callable[[int], None] = None) -> None: download_progress_callback: Callable[[int], None] = None) -> None:
super(ThreadedScraper, self).__init__(save_directory, files, super().__init__(save_directory, files, download_progress_callback)
download_progress_callback)
self._files_downloaded = 0 self._files_downloaded = 0
self._files_downloaded_mutex = Lock() self._files_downloaded_mutex = Lock()
@ -24,8 +25,8 @@ class ThreadedScraper(Scraper):
pool.join() pool.join()
def _thread_run(self, f: FileInfo): def _thread_run(self, f: FileInfo):
if not self._progress_callback is None:
with self._files_downloaded_mutex: with self._files_downloaded_mutex:
self._files_downloaded += 1 self._files_downloaded += 1
if not self._progress_callback is None:
self._progress_callback(self._files_downloaded) self._progress_callback(self._files_downloaded)
self._download_file(f) self._download_file(f)