Arav/ScrapTheChan
1
0
Fork 0
Code Releases 8 Activity

Lolifox removed. Added skip_posts handling.

Browse Source
This commit is contained in:
Alexander Andreev 2021-05-03 02:40:57 +04:00
parent 78d4a62c17
commit 1213cef776
Signed by: Arav
GPG Key ID: 610DF2574456329F
1 changed files with 11 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
scrapthechan/parsers/__init__.py
View File

@ -1,6 +1,6 @@
"""Here are defined the JSON parsers for imageboards."""
from re import search
from typing import List
from typing import List, Optional
from scrapthechan.parser import Parser
@ -8,33 +8,31 @@ from scrapthechan.parser import Parser
__all__ = ["SUPPORTED_IMAGEBOARDS", "get_parser_by_url", "get_parser_by_site"]
URLRX = r"https?:\/\/(?P<s>[\w\.]+)\/(?P<b>\w+)\/(?:\w+)?\/(?P<t>\w+)"
SUPPORTED_IMAGEBOARDS: List[str] = ["4chan.org", "lainchan.org", "2ch.hk", \
"8kun.top", "lolifox.cc"]
"8kun.top"]
def get_parser_by_url(url: str) -> Parser:
def get_parser_by_url(url: str, skip_posts: Optional[int] = None) -> Parser:
"""Parses URL and extracts from it site name, board and thread.
And then returns initialised Parser object for detected imageboard."""
URLRX = r"https?:\/\/(?P<s>[\w\.]+)\/(?P<b>\w+)\/(?:\w+)?\/(?P<t>\w+)"
site, board, thread = search(URLRX, url).groups()
return get_parser_by_site(site, board, thread)
return get_parser_by_site(site, board, thread, skip_posts)
def get_parser_by_site(site: str, board: str, thread: str) -> Parser:
def get_parser_by_site(site: str, board: str, thread: str,
skip_posts: Optional[int] = None) -> Parser:
"""Returns an initialised parser for `site` with `board` and `thread`."""
if '4chan' in site:
from .fourchan import FourChanParser
return FourChanParser(board, thread)
return FourChanParser(board, thread, skip_posts)
elif 'lainchan' in site:
from .lainchan import LainchanParser
return LainchanParser(board, thread)
return LainchanParser(board, thread, skip_posts)
elif '2ch' in site:
from .dvach import DvachParser
return DvachParser(board, thread)
return DvachParser(board, thread, skip_posts)
elif '8kun' in site:
from .eightkun import EightKunParser
return EightKunParser(board, thread)
elif 'lolifox' in site:
from .lolifox import LolifoxParser
return LolifoxParser(board, thread)
return EightKunParser(board, thread, skip_posts)
else:
raise NotImplementedError(f"Parser for {site} is not implemented")