From 7825b531217ac4ab18e4a2e8694f84fdfff1aa26 Mon Sep 17 00:00:00 2001 From: "Alexander \"Arav\" Andreev" Date: Mon, 20 Jul 2020 04:32:30 +0400 Subject: [PATCH] Did a minor refactoring. Also combined two first lines that are printed for a thread into one. --- scrapthechan/cli/scraper.py | 26 ++++++++++++-------------- scrapthechan/parser.py | 6 +++--- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/scrapthechan/cli/scraper.py b/scrapthechan/cli/scraper.py index e1c7ed9..41f3745 100644 --- a/scrapthechan/cli/scraper.py +++ b/scrapthechan/cli/scraper.py @@ -6,10 +6,9 @@ from sys import argv from typing import List from scrapthechan import VERSION -from scrapthechan.parser import Parser, ParserThreadNotFoundError +from scrapthechan.parser import Parser, ThreadNotFoundError from scrapthechan.parsers import get_parser_by_url, get_parser_by_site, \ SUPPORTED_IMAGEBOARDS -#from scrapthechan.scrapers.basicscraper import BasicScraper from scrapthechan.scrapers.threadedscraper import ThreadedScraper @@ -40,12 +39,12 @@ Supported imageboards: {', '.join(SUPPORTED_IMAGEBOARDS)}. def parse_common_arguments(args: str) -> dict: r = r"(?P-h|--help)|(?P-v|--version)" - argd = search(r, args) - if not argd is None: - argd = argd.groupdict() + args = search(r, args) + if not args is None: + args = args.groupdict() return { - "help": not argd["help"] is None, - "version": not argd["version"] is None } + "help": not args["help"] is None, + "version": not args["version"] is None } return None def parse_arguments(args: str) -> dict: @@ -84,16 +83,13 @@ def main() -> None: print(f"{str(ex)}.") print(f"Supported image boards are {', '.join(SUPPORTED_IMAGEBOARDS)}") exit() - except ParserThreadNotFoundError: + except ThreadNotFoundError: print(f"Thread {args['site']}/{args['board']}/{args['thread']} " \ "is no longer exist.") exit() - flen = len(parser.files) - - print(f"There are {flen} files in " \ - f"{args['site']}/{args['board']}/{args['thread']}.") + files_count = len(parser.files) if not args["output-dir"] is None: save_dir = args["output-dir"] @@ -101,7 +97,9 @@ def main() -> None: save_dir = join(parser.imageboard, parser.board, parser.thread) - print(f"They will be saved in {save_dir}.") + print(f"There are {files_count} files in " \ + f"{args['site']}/{args['board']}/{args['thread']}." \ + f"They will be saved in {save_dir}.") makedirs(save_dir, exist_ok=True) @@ -119,7 +117,7 @@ def main() -> None: scraper = ThreadedScraper(save_dir, parser.files, \ - lambda i: print(f"{i}/{flen}", end="\r")) + lambda i: print(f"{i}/{files_count}", end="\r")) scraper.run() diff --git a/scrapthechan/parser.py b/scrapthechan/parser.py index 014a009..7a23bd9 100644 --- a/scrapthechan/parser.py +++ b/scrapthechan/parser.py @@ -9,10 +9,10 @@ from urllib.request import urlopen, urlretrieve from scrapthechan.fileinfo import FileInfo -__all__ = ["Parser", "ParserThreadNotFoundError"] +__all__ = ["Parser", "ThreadNotFoundError"] -class ParserThreadNotFoundError(Exception): +class ThreadNotFoundError(Exception): pass @@ -74,7 +74,7 @@ class Parser: with urlopen(thread_url) as url: return loads(url.read().decode('utf-8')) except: - raise ParserThreadNotFoundError + raise ThreadNotFoundError def _parse_post(self, post: dict) -> List[FileInfo]: """Parses a single post and extracts files into `FileInfo` object."""