Arav/ScrapTheChan
1
0
Fork 0
Code Releases 8 Activity

BasicScraper is replaced with a threaded one. Also OP's post handling altered.

Browse Source
This commit is contained in:
Alexander Andreev 2020-07-18 04:44:37 +04:00
parent 195d4d057a
commit 22961611da
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapthechan/cli/scraper.py
View File

@ -9,7 +9,8 @@ from scrapthechan import VERSION
from scrapthechan.parser import Parser, ParserThreadNotFoundError from scrapthechan.parser import Parser, ParserThreadNotFoundError
from scrapthechan.parsers import get_parser_by_url, get_parser_by_site, \ from scrapthechan.parsers import get_parser_by_url, get_parser_by_site, \
SUPPORTED_IMAGEBOARDS SUPPORTED_IMAGEBOARDS
from scrapthechan.scrapers.basicscraper import BasicScraper #from scrapthechan.scrapers.basicscraper import BasicScraper
from scrapthechan.scrapers.threadedscraper import ThreadedScraper
__all__ = ["main"] __all__ = ["main"]
@ -105,7 +106,9 @@ def main() -> None:
if not args["no-op"]: if not args["no-op"]:
print("Writing OP... ", end='') print("Writing OP... ", end='')
if not exists(join(save_dir, "!op.txt")): if parser.op is None:
print("No text's there.")
elif not exists(join(save_dir, "!op.txt")):
with open(join(save_dir, "!op.txt"), 'w') as opf: with open(join(save_dir, "!op.txt"), 'w') as opf:
opf.write(f"{parser.op}\n") opf.write(f"{parser.op}\n")
print("Done.") print("Done.")
@ -113,7 +116,7 @@ def main() -> None:
print("Exists.") print("Exists.")
scraper = BasicScraper(save_dir, parser.files, \ scraper = ThreadedScraper(save_dir, parser.files, \
lambda i: print(f"{i}/{flen}", end="\r")) lambda i: print(f"{i}/{flen}", end="\r"))
scraper.run() scraper.run()