Code formatting and updated flamecomic to use changed api
This commit is contained in:
@@ -1,31 +1,30 @@
|
|||||||
import requests
|
|
||||||
from pathlib import Path
|
|
||||||
import argparse
|
import argparse
|
||||||
import configparser
|
import configparser
|
||||||
import json
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import questionary
|
import questionary
|
||||||
from rich import print
|
import requests
|
||||||
|
|
||||||
from sites import parse_provider
|
|
||||||
from manga import Manga
|
from manga import Manga
|
||||||
|
from rich import print
|
||||||
|
from sites import parse_provider
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
prog="Manga Downloader",
|
prog="Manga Downloader",
|
||||||
description="Download all your favourite manga!",
|
description="Download all your favourite manga!",
|
||||||
epilog="Happy Reading!"
|
epilog="Happy Reading!",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--auto",
|
"--auto",
|
||||||
default=False,
|
default=False,
|
||||||
action=argparse.BooleanOptionalAction,
|
action=argparse.BooleanOptionalAction,
|
||||||
help="Run using auto mode (default: %(default)s)"
|
help="Run using auto mode (default: %(default)s)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--save-dir",
|
"--save-dir",
|
||||||
dest="save_dir",
|
dest="save_dir",
|
||||||
type=Path,
|
type=Path,
|
||||||
help="Set temporary save dir instead of using config value"
|
help="Set temporary save dir instead of using config value",
|
||||||
)
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
@@ -41,7 +40,9 @@ def get_config() -> configparser.ConfigParser:
|
|||||||
config.read(config_file)
|
config.read(config_file)
|
||||||
|
|
||||||
if not config.has_option("DEFAULT", "savedir"):
|
if not config.has_option("DEFAULT", "savedir"):
|
||||||
path = questionary.path("Enter default save path for downloaded Manga: ", qmark="").ask()
|
path = questionary.path(
|
||||||
|
"Enter default save path for downloaded Manga: ", qmark=""
|
||||||
|
).ask()
|
||||||
config["DEFAULT"] = {"savedir": path}
|
config["DEFAULT"] = {"savedir": path}
|
||||||
|
|
||||||
with open(config_file, "w") as cf:
|
with open(config_file, "w") as cf:
|
||||||
@@ -49,25 +50,25 @@ def get_config() -> configparser.ConfigParser:
|
|||||||
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
|
||||||
def read_from_cache() -> dict:
|
def read_from_cache() -> dict:
|
||||||
if cache_file.exists():
|
if cache_file.exists():
|
||||||
return json.loads(cache_file.read_bytes())
|
return json.loads(cache_file.read_bytes())
|
||||||
else:
|
else:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def write_to_cache(manga: Manga) -> None:
|
def write_to_cache(manga: Manga) -> None:
|
||||||
cache = read_from_cache()
|
cache = read_from_cache()
|
||||||
series = manga.info.get("series")
|
series = manga.info.get("series")
|
||||||
cached_chapters = cache.get(series, {}).get("cached", []) + manga.cached
|
cached_chapters = cache.get(series, {}).get("cached", []) + manga.cached
|
||||||
|
|
||||||
cache[series] = {
|
cache[series] = {"url": manga.site.get_url(), "cached": list(set(cached_chapters))}
|
||||||
"url": manga.site.get_url(),
|
|
||||||
"cached": list(set(cached_chapters))
|
|
||||||
}
|
|
||||||
|
|
||||||
with open(cache_file, "w") as fp:
|
with open(cache_file, "w") as fp:
|
||||||
json.dump(cache, fp, indent=4)
|
json.dump(cache, fp, indent=4)
|
||||||
|
|
||||||
|
|
||||||
def parse_cache(cache: dict) -> list[Manga]:
|
def parse_cache(cache: dict) -> list[Manga]:
|
||||||
if not cache:
|
if not cache:
|
||||||
print("Cache is empty!")
|
print("Cache is empty!")
|
||||||
@@ -80,24 +81,24 @@ def parse_cache(cache:dict) -> list[Manga]:
|
|||||||
)
|
)
|
||||||
return manga
|
return manga
|
||||||
|
|
||||||
|
|
||||||
def get_cached_manga() -> list[Manga]:
|
def get_cached_manga() -> list[Manga]:
|
||||||
cache = read_from_cache()
|
cache = read_from_cache()
|
||||||
return parse_cache(cache)
|
return parse_cache(cache)
|
||||||
|
|
||||||
|
|
||||||
def input_manga() -> list[str]:
|
def input_manga() -> list[str]:
|
||||||
manga = list()
|
manga = list()
|
||||||
add_more = True
|
add_more = True
|
||||||
while add_more:
|
while add_more:
|
||||||
manga.append(
|
manga.append(questionary.text("Input Manga url:", qmark="").ask())
|
||||||
questionary.text(
|
add_more = questionary.confirm(
|
||||||
"Input Manga url:",
|
"Do you want to add more Manga", default=False
|
||||||
qmark = ""
|
|
||||||
).ask()
|
).ask()
|
||||||
)
|
|
||||||
add_more = questionary.confirm("Do you want to add more Manga", default=False).ask()
|
|
||||||
|
|
||||||
return manga
|
return manga
|
||||||
|
|
||||||
|
|
||||||
def parse_manga(manga: list[str]) -> list[Manga]:
|
def parse_manga(manga: list[str]) -> list[Manga]:
|
||||||
parsed_manga = list()
|
parsed_manga = list()
|
||||||
for entry in manga:
|
for entry in manga:
|
||||||
@@ -108,8 +109,11 @@ def parse_manga(manga:list[str]) -> list[Manga]:
|
|||||||
parsed_manga.append(Manga(provider))
|
parsed_manga.append(Manga(provider))
|
||||||
return parsed_manga
|
return parsed_manga
|
||||||
|
|
||||||
|
|
||||||
def choose_manga() -> list[Manga]:
|
def choose_manga() -> list[Manga]:
|
||||||
use_cached = questionary.confirm("Do you want to choose from cached Manga", default=True).ask()
|
use_cached = questionary.confirm(
|
||||||
|
"Do you want to choose from cached Manga", default=True
|
||||||
|
).ask()
|
||||||
|
|
||||||
if use_cached:
|
if use_cached:
|
||||||
return get_cached_manga()
|
return get_cached_manga()
|
||||||
@@ -117,6 +121,7 @@ def choose_manga() -> list[Manga]:
|
|||||||
manga = input_manga()
|
manga = input_manga()
|
||||||
return parse_manga(manga)
|
return parse_manga(manga)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
||||||
config = get_config()
|
config = get_config()
|
||||||
@@ -136,5 +141,6 @@ def main():
|
|||||||
manga.download(save_dir)
|
manga.download(save_dir)
|
||||||
write_to_cache(manga)
|
write_to_cache(manga)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
|
import tempfile
|
||||||
import xml.etree.ElementTree as XML
|
import xml.etree.ElementTree as XML
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import tempfile
|
|
||||||
|
|
||||||
import questionary
|
import questionary
|
||||||
from rich import print
|
|
||||||
from rich.progress import Progress, SpinnerColumn, TextColumn, MofNCompleteColumn
|
|
||||||
|
|
||||||
from sites import Provider
|
|
||||||
from packager import download_images, zip_files
|
from packager import download_images, zip_files
|
||||||
|
from rich import print
|
||||||
|
from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn, TextColumn
|
||||||
|
from sites import Provider
|
||||||
|
|
||||||
class Manga():
|
|
||||||
|
class Manga:
|
||||||
def __init__(self, provider: Provider, cached: list[int] = []) -> None:
|
def __init__(self, provider: Provider, cached: list[int] = []) -> None:
|
||||||
self.site = provider
|
self.site = provider
|
||||||
self.cached = cached
|
self.cached = cached
|
||||||
@@ -34,7 +34,9 @@ class Manga():
|
|||||||
XML.SubElement(ComicInfo, "Title").text = chapter.get("title")
|
XML.SubElement(ComicInfo, "Title").text = chapter.get("title")
|
||||||
XML.SubElement(ComicInfo, "LanguageISO").text = self.info.get("languageISO")
|
XML.SubElement(ComicInfo, "LanguageISO").text = self.info.get("languageISO")
|
||||||
XML.SubElement(ComicInfo, "PageCount").text = str(len(chapter.get("images"))) # type: ignore
|
XML.SubElement(ComicInfo, "PageCount").text = str(len(chapter.get("images"))) # type: ignore
|
||||||
XML.SubElement(ComicInfo, "ScanInformation").text = self.info.get("scanInformation")
|
XML.SubElement(ComicInfo, "ScanInformation").text = self.info.get(
|
||||||
|
"scanInformation"
|
||||||
|
)
|
||||||
|
|
||||||
tree = XML.ElementTree(ComicInfo)
|
tree = XML.ElementTree(ComicInfo)
|
||||||
|
|
||||||
@@ -50,18 +52,24 @@ class Manga():
|
|||||||
def choose_chapters(self, auto):
|
def choose_chapters(self, auto):
|
||||||
choices = [
|
choices = [
|
||||||
questionary.Choice(
|
questionary.Choice(
|
||||||
f"{chapter.get("nr"):02d}: {chapter.get("title")}",
|
f"{chapter.get('nr'):02d}: {chapter.get('title')}",
|
||||||
chapter,
|
chapter,
|
||||||
checked = chapter.get("nr") not in self.cached
|
checked=chapter.get("nr") not in self.cached,
|
||||||
) for chapter in self.chapters
|
)
|
||||||
|
for chapter in self.chapters
|
||||||
]
|
]
|
||||||
self.chapters = questionary.checkbox(
|
self.chapters = (
|
||||||
"Choose which chapters to download",
|
questionary.checkbox("Choose which chapters to download", choices)
|
||||||
choices
|
.skip_if(
|
||||||
).skip_if(
|
|
||||||
auto,
|
auto,
|
||||||
[chapter for chapter in self.chapters if chapter.get("nr") not in self.cached]
|
[
|
||||||
).ask()
|
chapter
|
||||||
|
for chapter in self.chapters
|
||||||
|
if chapter.get("nr") not in self.cached
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.ask()
|
||||||
|
)
|
||||||
|
|
||||||
def download(self, save_dir: Path):
|
def download(self, save_dir: Path):
|
||||||
with tempfile.TemporaryDirectory(prefix="manga_") as temp_dir:
|
with tempfile.TemporaryDirectory(prefix="manga_") as temp_dir:
|
||||||
@@ -69,12 +77,17 @@ class Manga():
|
|||||||
with Progress(
|
with Progress(
|
||||||
TextColumn("[progress.description]{task.description}"),
|
TextColumn("[progress.description]{task.description}"),
|
||||||
MofNCompleteColumn(),
|
MofNCompleteColumn(),
|
||||||
SpinnerColumn()
|
SpinnerColumn(),
|
||||||
) as p:
|
) as p:
|
||||||
task = p.add_task(f"[bold]Processing [orange3]{self.info.get("series")}[/]", total=len(self.chapters))
|
task = p.add_task(
|
||||||
|
f"[bold]Processing [orange3]{self.info.get('series')}[/]",
|
||||||
|
total=len(self.chapters),
|
||||||
|
)
|
||||||
while not p.finished:
|
while not p.finished:
|
||||||
for chapter in self.chapters:
|
for chapter in self.chapters:
|
||||||
print(f"Downloading [bold blue]Chapter {chapter.get("nr"):02d}[/]...")
|
print(
|
||||||
|
f"Downloading [bold blue]Chapter {chapter.get('nr'):02d}[/]..."
|
||||||
|
)
|
||||||
files = [self._generateComicInfo(chapter, Path(temp_dir))]
|
files = [self._generateComicInfo(chapter, Path(temp_dir))]
|
||||||
files.extend(download_images(chapter.get("images"), Path(temp_dir))) # type: ignore
|
files.extend(download_images(chapter.get("images"), Path(temp_dir))) # type: ignore
|
||||||
zip_files(self.slug, chapter, files, save_dir)
|
zip_files(self.slug, chapter, files, save_dir)
|
||||||
|
|||||||
+14
-6
@@ -1,11 +1,15 @@
|
|||||||
from pathlib import Path
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import aiohttp
|
|
||||||
import aiofiles
|
|
||||||
from aiohttp import ClientError
|
|
||||||
import zipfile
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
async def download_image(session:aiohttp.ClientSession, url:str, temp_dir:Path, retries=3, timeout=10):
|
import aiofiles
|
||||||
|
import aiohttp
|
||||||
|
from aiohttp import ClientError
|
||||||
|
|
||||||
|
|
||||||
|
async def download_image(
|
||||||
|
session: aiohttp.ClientSession, url: str, temp_dir: Path, retries=3, timeout=10
|
||||||
|
):
|
||||||
|
|
||||||
filename = temp_dir.joinpath(Path(url).name)
|
filename = temp_dir.joinpath(Path(url).name)
|
||||||
|
|
||||||
@@ -25,10 +29,12 @@ async def download_image(session:aiohttp.ClientSession, url:str, temp_dir:Path,
|
|||||||
print(f"Failed to download {url}: {e}")
|
print(f"Failed to download {url}: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
async def _download_images(urls, temp_dir: Path, concurrency=5):
|
async def _download_images(urls, temp_dir: Path, concurrency=5):
|
||||||
semaphore = asyncio.Semaphore(concurrency)
|
semaphore = asyncio.Semaphore(concurrency)
|
||||||
|
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
|
|
||||||
async def bounded_download(url):
|
async def bounded_download(url):
|
||||||
async with semaphore:
|
async with semaphore:
|
||||||
return await download_image(session, url, temp_dir)
|
return await download_image(session, url, temp_dir)
|
||||||
@@ -36,12 +42,14 @@ async def _download_images(urls, temp_dir:Path, concurrency=5):
|
|||||||
tasks = [bounded_download(url) for url in urls]
|
tasks = [bounded_download(url) for url in urls]
|
||||||
return await asyncio.gather(*tasks)
|
return await asyncio.gather(*tasks)
|
||||||
|
|
||||||
|
|
||||||
def download_images(images: list[str], temp_dir: Path):
|
def download_images(images: list[str], temp_dir: Path):
|
||||||
return asyncio.run(_download_images(images, temp_dir))
|
return asyncio.run(_download_images(images, temp_dir))
|
||||||
|
|
||||||
|
|
||||||
def zip_files(slug, chapter: dict, files: list[Path], save_dir: Path):
|
def zip_files(slug, chapter: dict, files: list[Path], save_dir: Path):
|
||||||
save_dir.joinpath(slug).mkdir(exist_ok=True)
|
save_dir.joinpath(slug).mkdir(exist_ok=True)
|
||||||
filename = save_dir.joinpath(slug).joinpath(f"{slug} - Ch. {chapter.get("nr")}.cbz") # type: ignore
|
filename = save_dir.joinpath(slug).joinpath(f"{slug} - Ch. {chapter.get('nr')}.cbz") # type: ignore
|
||||||
with zipfile.ZipFile(filename, "w") as zf:
|
with zipfile.ZipFile(filename, "w") as zf:
|
||||||
for file in files:
|
for file in files:
|
||||||
zf.write(file, file.name)
|
zf.write(file, file.name)
|
||||||
|
|||||||
+1
-1
@@ -5,8 +5,8 @@ PROVIDERS = [
|
|||||||
FlameComics,
|
FlameComics,
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def parse_provider(session, url) -> Provider | None:
|
def parse_provider(session, url) -> Provider | None:
|
||||||
for site in PROVIDERS:
|
for site in PROVIDERS:
|
||||||
if site.domain() in url:
|
if site.domain() in url:
|
||||||
return site(session, url)
|
return site(session, url)
|
||||||
|
|
||||||
|
|||||||
+28
-19
@@ -1,13 +1,17 @@
|
|||||||
import requests
|
|
||||||
import json
|
import json
|
||||||
from bs4 import BeautifulSoup as bs
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
import iso639
|
import iso639
|
||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup as bs
|
||||||
|
|
||||||
from .provider import Provider
|
from .provider import Provider
|
||||||
|
|
||||||
CDN_URL = "https://cdn.flamecomics.xyz"
|
CDN_URL = "https://cdn.flamecomics.xyz"
|
||||||
IMAGE_URL = CDN_URL + "/uploads/images/series/{id}/{token}/{file}"
|
IMAGE_URL = CDN_URL + "/uploads/images/series/{id}/{token}/{file}"
|
||||||
INFO_URL = "https://flamecomics.xyz/series/{id}"
|
INFO_URL = "https://flamecomics.xyz/series/{id}"
|
||||||
|
CHAPTER_URL = "https://flamecomics.xyz/series/{id}/{token}"
|
||||||
|
|
||||||
|
|
||||||
class FlameComics(Provider):
|
class FlameComics(Provider):
|
||||||
|
|
||||||
@@ -26,8 +30,12 @@ class FlameComics(Provider):
|
|||||||
"writer": payload["series"].get("author"),
|
"writer": payload["series"].get("author"),
|
||||||
"penciller": payload["series"].get("artist"),
|
"penciller": payload["series"].get("artist"),
|
||||||
"genre": payload["series"].get("tags"),
|
"genre": payload["series"].get("tags"),
|
||||||
"summary": bs(payload["series"].get("description"), "html.parser").get_text(),
|
"summary": bs(
|
||||||
"languageISO": iso639.Language.from_name(payload["series"].get("language")).part1,
|
payload["series"].get("description"), "html.parser"
|
||||||
|
).get_text(),
|
||||||
|
"languageISO": iso639.Language.from_name(
|
||||||
|
payload["series"].get("language")
|
||||||
|
).part1,
|
||||||
"scanInformation": "Reaper_Scans & Flame Comics",
|
"scanInformation": "Reaper_Scans & Flame Comics",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -46,28 +54,31 @@ class FlameComics(Provider):
|
|||||||
title = c.get("title")
|
title = c.get("title")
|
||||||
if not title:
|
if not title:
|
||||||
title = f"Chapter {nbr}"
|
title = f"Chapter {nbr}"
|
||||||
images = self.generate_image_urls(c.get("images"), c.get("token")) # type: ignore
|
r = self.s.get(CHAPTER_URL.format(id=self.id, token=c.get("token")))
|
||||||
|
chapter_page = bs(r.content, "html.parser")
|
||||||
|
chapter_payload = self.get_page_props(chapter_page).get("chapter")
|
||||||
|
images = self.generate_image_urls(chapter_payload.get("images"), chapter_payload.get("token")) # type: ignore
|
||||||
|
|
||||||
data = {
|
data = {"nr": nbr, "title": title, "images": images}
|
||||||
"nr": nbr,
|
|
||||||
"title": title,
|
|
||||||
"images": images
|
|
||||||
}
|
|
||||||
|
|
||||||
chapters.append(data)
|
chapters.append(data)
|
||||||
|
|
||||||
return chapters
|
return chapters
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_page_props(page: bs) -> dict:
|
||||||
|
return (
|
||||||
|
json.loads(
|
||||||
|
page.find("script", attrs={"id": "__NEXT_DATA__"}).text # type: ignore
|
||||||
|
)
|
||||||
|
.get("props")
|
||||||
|
.get("pageProps")
|
||||||
|
)
|
||||||
|
|
||||||
def get_info(self) -> tuple[dict, list]:
|
def get_info(self) -> tuple[dict, list]:
|
||||||
r = self.s.get(INFO_URL.format(id=self.id))
|
r = self.s.get(INFO_URL.format(id=self.id))
|
||||||
series_page = bs(r.content, "html.parser")
|
series_page = bs(r.content, "html.parser")
|
||||||
payload = json.loads(
|
payload = self.get_page_props(series_page)
|
||||||
series_page.find(
|
|
||||||
"script",
|
|
||||||
attrs = {"id": "__NEXT_DATA__"}
|
|
||||||
).text # type: ignore
|
|
||||||
).get("props").get("pageProps")
|
|
||||||
|
|
||||||
info = self.parse_info(payload)
|
info = self.parse_info(payload)
|
||||||
chapters = self.parse_chapters(payload["chapters"])
|
chapters = self.parse_chapters(payload["chapters"])
|
||||||
return info, chapters
|
return info, chapters
|
||||||
@@ -81,5 +92,3 @@ class FlameComics(Provider):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def domain() -> str:
|
def domain() -> str:
|
||||||
return "flamecomics.xyz"
|
return "flamecomics.xyz"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -1,5 +1,6 @@
|
|||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
|
||||||
class Provider(ABC):
|
class Provider(ABC):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -14,4 +15,3 @@ class Provider(ABC):
|
|||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_url(self) -> str:
|
def get_url(self) -> str:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
Reference in New Issue
Block a user