Arav/ScrapTheChan
1
0
Fork 0
Code Releases 8 Activity

Moved User-Agent off to __init__ in its own variable.

Browse Source
This commit is contained in:
Alexander Andreev 2020-07-20 04:31:27 +04:00
parent 9ad9fcfd6f
commit b26152f3ca
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapthechan/__init__.py
View File

@ -8,6 +8,9 @@ __license__ = \
For a copy see COPYING file in a directory of the program, or For a copy see COPYING file in a directory of the program, or
see <https://opensource.org/licenses/MIT>""" see <https://opensource.org/licenses/MIT>"""
USER_AGENT = f"ScrapTheChan/{__version__}"
VERSION = \ VERSION = \
f"ScrapTheChan ver. {__version__} ({__date__})\n{__copyright__}\n"\ f"ScrapTheChan ver. {__version__} ({__date__})\n{__copyright__}\n"\
f"\n{__license__}" f"\n{__license__}"

8
scrapthechan/scraper.py
View File

@ -1,4 +1,4 @@
"""Base Scraper implementation.""" """Base class for all scrapers that will actually do the job."""
from base64 import b64encode from base64 import b64encode
from os import remove, stat from os import remove, stat
@ -8,14 +8,14 @@ from typing import List, Callable
from urllib.request import urlretrieve, URLopener from urllib.request import urlretrieve, URLopener
import hashlib import hashlib
from scrapthechan import __version__ from scrapthechan import USER_AGENT
from scrapthechan.fileinfo import FileInfo from scrapthechan.fileinfo import FileInfo
__all__ = ["Scraper"] __all__ = ["Scraper"]
class Scraper: class Scraper:
"""Base scraper implementation. """Base class for all scrapers that will actually do the job.
Arguments: Arguments:
save_directory -- a path to a directory where file will be save_directory -- a path to a directory where file will be
@ -29,7 +29,7 @@ class Scraper:
self._save_directory = save_directory self._save_directory = save_directory
self._files = files self._files = files
self._url_opener = URLopener() self._url_opener = URLopener()
self._url_opener.version = f"ScrapTheChan/{__version__}" self._url_opener.version = USER_AGENT
self._progress_callback = download_progress_callback self._progress_callback = download_progress_callback
def run(self): def run(self):