diff --git a/manga.py b/manga.py index fd90e4b..1a49f4f 100644 --- a/manga.py +++ b/manga.py @@ -1,19 +1,29 @@ +import asyncio import tempfile import xml.etree.ElementTree as XML from pathlib import Path +import aiofiles +import aiohttp import questionary -from packager import download_images, zip_files -from rich import print +from aiohttp import ClientError +from rich import print, status from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn, TextColumn + +from packager import zip_files from sites import Provider class Manga: - def __init__(self, provider: Provider, cached: list[int] = []) -> None: + def __init__(self, provider: Provider, cached: list[float] = []) -> None: self.site = provider self.cached = cached + + spinner = status.Status("Fetching manga info") + spinner.start() self.info, self.chapters = self.site.get_mediainfo() + spinner.stop() + self.slug = self.slugify(self.info.get("series")) # type: ignore @staticmethod @@ -26,9 +36,9 @@ class Manga: ComicInfo = XML.Element("ComicInfo") XML.SubElement(ComicInfo, "Series").text = self.info.get("series") - XML.SubElement(ComicInfo, "Writer").text = ",".join(self.info.get("writer")) # type: ignore - XML.SubElement(ComicInfo, "Penciller").text = ",".join(self.info.get("penciller")) # type: ignore - XML.SubElement(ComicInfo, "Genre").text = ",".join(self.info.get("genre")) # type: ignore + XML.SubElement(ComicInfo, "Writer").text = ",".join(self.info.get("writer", [])) # type: ignore + XML.SubElement(ComicInfo, "Penciller").text = ",".join(self.info.get("penciller", [])) # type: ignore + XML.SubElement(ComicInfo, "Genre").text = ",".join(self.info.get("genre", [])) # type: ignore XML.SubElement(ComicInfo, "Summary").text = self.info.get("summary") XML.SubElement(ComicInfo, "Number").text = str(chapter.get("nr")) XML.SubElement(ComicInfo, "Title").text = chapter.get("title") @@ -52,7 +62,7 @@ class Manga: def choose_chapters(self, auto): choices = [ questionary.Choice( - f"{chapter.get('nr'):02d}: {chapter.get('title')}", + f"{chapter.get('nr'):g}: {chapter.get('title')}", chapter, checked=chapter.get("nr") not in self.cached, ) @@ -72,8 +82,19 @@ class Manga: ) def download(self, save_dir: Path): - with tempfile.TemporaryDirectory(prefix="manga_") as temp_dir: - if self.chapters: + if not self.chapters: + print("No new [orange3]Chapter[/] were found...") + return + + async def process_all_chapters(): + + chapter_sem = asyncio.Semaphore(8) + image_sem = asyncio.Semaphore(40) + + connector = aiohttp.TCPConnector(limit=100, ttl_dns_cache=300) + async with aiohttp.ClientSession( + headers=self.site.headers, connector=connector + ) as session: with Progress( TextColumn("[progress.description]{task.description}"), MofNCompleteColumn(), @@ -83,15 +104,71 @@ class Manga: f"[bold]Processing [orange3]{self.info.get('series')}[/]", total=len(self.chapters), ) - while not p.finished: - for chapter in self.chapters: - print( - f"Downloading [bold blue]Chapter {chapter.get('nr'):02d}[/]..." + + async def download_and_zip(chapter): + async with chapter_sem: + chapter_nr = chapter.get("nr") + chapter_temp = Path(temp_dir) / f"ch_{chapter_nr:g}" + chapter_temp.mkdir(parents=True, exist_ok=True) + info_file = self._generateComicInfo(chapter, chapter_temp) + img_urls = chapter.get("images", []) + img_files = await self._async_download_images_internal( + img_urls, chapter_temp, session, image_sem ) - files = [self._generateComicInfo(chapter, Path(temp_dir))] - files.extend(download_images(chapter.get("images"), Path(temp_dir))) # type: ignore - zip_files(self.slug, chapter, files, save_dir) - self.cached.append(chapter.get("nr")) + + all_files = [info_file] + [f for f in img_files if f] + zip_files(self.slug, chapter, all_files, save_dir) + + self.cached.append(chapter_nr) p.update(task, advance=1) - else: - print("No new [orange3]Chapter[/] were found...") + + await asyncio.gather( + *(download_and_zip(ch) for ch in self.chapters) + ) + + with tempfile.TemporaryDirectory(prefix="manga_") as temp_dir: + asyncio.run(process_all_chapters()) + + async def _async_download_images_internal(self, urls, temp_dir, session, semaphore): + """Worker to manage the image download tasks for a single chapter.""" + + async def bounded_download(url): + async with semaphore: + return await self.download_image( + session, url, temp_dir, self.site.headers + ) + + tasks = [bounded_download(url) for url in urls] + return await asyncio.gather(*tasks) + + @staticmethod + async def download_image( + session: aiohttp.ClientSession, + url: str, + temp_dir: Path, + headers: dict, + retries=3, + timeout=10, + ): + + filename = temp_dir.joinpath(Path(url).name) + + for attempt in range(1, retries + 1): + try: + async with session.get(url, headers=headers, timeout=timeout) as response: # type: ignore + if response.status == 200: + async with aiofiles.open(filename, "wb") as f: + await f.write(await response.read()) + return filename + else: + raise ClientError(f"Bad status {response.status} for {url}") + except ( + asyncio.TimeoutError, + ClientError, + aiohttp.ClientConnectorError, + ) as e: + if attempt < retries: + await asyncio.sleep(2**attempt) + else: + print(f"Failed to download {url}: {e}") + return None diff --git a/packager.py b/packager.py index 28b1c2a..7df55e1 100644 --- a/packager.py +++ b/packager.py @@ -1,55 +1,10 @@ -import asyncio import zipfile from pathlib import Path -import aiofiles -import aiohttp -from aiohttp import ClientError - - -async def download_image( - session: aiohttp.ClientSession, url: str, temp_dir: Path, retries=3, timeout=10 -): - - filename = temp_dir.joinpath(Path(url).name) - - for attempt in range(1, retries + 1): - try: - async with session.get(url, timeout=timeout) as response: # type: ignore - if response.status == 200: - async with aiofiles.open(filename, "wb") as f: - await f.write(await response.read()) - return filename - else: - raise ClientError(f"Bad status {response.status} for {url}") - except (asyncio.TimeoutError, ClientError, aiohttp.ClientConnectorError) as e: - if attempt < retries: - await asyncio.sleep(2**attempt) - else: - print(f"Failed to download {url}: {e}") - return None - - -async def _download_images(urls, temp_dir: Path, concurrency=5): - semaphore = asyncio.Semaphore(concurrency) - - async with aiohttp.ClientSession() as session: - - async def bounded_download(url): - async with semaphore: - return await download_image(session, url, temp_dir) - - tasks = [bounded_download(url) for url in urls] - return await asyncio.gather(*tasks) - - -def download_images(images: list[str], temp_dir: Path): - return asyncio.run(_download_images(images, temp_dir)) - def zip_files(slug, chapter: dict, files: list[Path], save_dir: Path): save_dir.joinpath(slug).mkdir(exist_ok=True) - filename = save_dir.joinpath(slug).joinpath(f"{slug} - Ch. {chapter.get('nr')}.cbz") # type: ignore + filename = save_dir.joinpath(slug).joinpath(f"{slug} - Ch. {chapter.get('nr'):g}.cbz") # type: ignore with zipfile.ZipFile(filename, "w") as zf: for file in files: zf.write(file, file.name) diff --git a/requirements.txt b/requirements.txt index b886f31..8c3ca65 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/sites/__init__.py b/sites/__init__.py index 87cf669..3930a83 100644 --- a/sites/__init__.py +++ b/sites/__init__.py @@ -1,12 +1,11 @@ from .flamecomics import FlameComics +from .mangapill import Mangapill from .provider import Provider -PROVIDERS = [ - FlameComics, -] +PROVIDERS = [FlameComics, Mangapill] def parse_provider(session, url) -> Provider | None: for site in PROVIDERS: - if site.domain() in url: + if site.domain in url: return site(session, url) diff --git a/sites/flamecomics.py b/sites/flamecomics.py index 71a8ac9..98e21e1 100644 --- a/sites/flamecomics.py +++ b/sites/flamecomics.py @@ -15,6 +15,15 @@ CHAPTER_URL = "https://flamecomics.xyz/series/{id}/{token}" class FlameComics(Provider): + domain = "flamecomics.xyz" + headers = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ) + } + def __init__(self, session: requests.Session, url) -> None: self.s = session self.id = self.parse_url(url) @@ -50,7 +59,7 @@ class FlameComics(Provider): def parse_chapters(self, payload: list[dict]) -> list[dict]: chapters = list() for c in payload: - nbr = int(float(c.get("chapter", 0.0))) + nbr = float(c.get("chapter", 0.0)) title = c.get("title") if not title: title = f"Chapter {nbr}" @@ -88,7 +97,3 @@ class FlameComics(Provider): def get_url(self) -> str: return INFO_URL.format(id=self.id) - - @staticmethod - def domain() -> str: - return "flamecomics.xyz" diff --git a/sites/mangapill.py b/sites/mangapill.py new file mode 100644 index 0000000..50c36db --- /dev/null +++ b/sites/mangapill.py @@ -0,0 +1,136 @@ +import asyncio +import re +from typing import TYPE_CHECKING + +import aiohttp +from bs4 import BeautifulSoup as bs + +from .provider import Provider + +if TYPE_CHECKING: + import requests + +BASE_URL = "https://mangapill.com" +INFO_URL = "https://mangapill.com/manga/{id}" + +# CDN_URL = "https://cdn.readdetectiveconan.com" +# IMAGE_URL = CDN_URL + "/file/mangap/{id}/10{chapter_nr:03d}000/{file}" +# CHAPTER_URL = "https://mangapill.com/chapters/{id}-10{chapter_nr:03d}000" + + +class Mangapill(Provider): + domain = "mangapill.com" + headers = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ), + "Referer": "https://mangapill.com/", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + } + + def __init__(self, session: "requests.Session", url) -> None: + self.s = session + self.id = self.parse_url(url) + + def parse_url(self, url: str) -> str: + parse = re.search(r"manga\/(?P\d+)", url) + return parse.group("id") if parse else "" + + def get_url(self) -> str: + return INFO_URL.format(id=self.id) + + @staticmethod + def parse_info(payload: bs) -> dict: + title_elem = payload.select_one("h1.font-bold") + title = title_elem.get_text(strip=True) if title_elem else "Unknown Title" + desc_p = payload.select_one("p.text-sm.text--secondary") + description = None + + if desc_p: + html_content = desc_p.decode_contents() + if "

" in html_content: + html_content = html_content.split("

", 1)[1] + clean_soup = bs(html_content, "html.parser") + text = clean_soup.get_text("\n", strip=True) + lines = [line.strip() for line in text.split("\n") if line.strip()] + for i, line in enumerate(lines): + if line.startswith("The "): + lines = lines[i:] + break + description = "\n\n".join(lines) if lines else None + + genres = [ + a.get_text(strip=True) for a in payload.select("a[href^='/search?genre=']") + ] + + return { + "series": title, + "genre": genres, + "summary": description, + "scanInformation": "MangaPill", + } + + async def _fetch_chapter( + self, + session: aiohttp.ClientSession, + href: str, + title: str, + nbr: float, + semaphore: asyncio.Semaphore, + ): + """Internal async worker to fetch a single chapter's images.""" + async with semaphore: + try: + async with session.get(BASE_URL + href) as response: + content = await response.read() + soup = bs(content, "html.parser") + + images = [] + for img in soup.select("img.js-page"): + src = img.get("data-src") or img.get("src") + if src: + images.append(src) + + return {"nr": nbr, "title": title, "images": images} + except Exception: + return {"nr": nbr, "title": title, "images": []} + + async def _async_get_all_chapters(self, chapter_tags: list) -> list: + """Sets up the concurrent tasks and executes them.""" + semaphore = asyncio.Semaphore(25) # Limits to 15 concurrent requests + total_ch = len(chapter_tags) + + async with aiohttp.ClientSession(headers=self.headers) as session: + tasks = [] + for i, a in enumerate(chapter_tags): + chapter_title = a.get_text(strip=True) + match = re.search( + r"Chapter\s*(?P[\d.]+)", chapter_title, re.IGNORECASE + ) + nbr = float(match.group("nbr")) if match else float(total_ch - i) + + tasks.append( + self._fetch_chapter( + session, a["href"], chapter_title, nbr, semaphore + ) + ) + + return await asyncio.gather(*tasks) + + def get_mediainfo(self) -> tuple[dict, list]: + + r = self.s.get(self.get_url()) + series_page = bs(r.content, "html.parser") + + info = self.parse_info(series_page) + + raw_chapters = series_page.select("#chapters a[href^='/chapters/']") + + chapters = asyncio.run(self._async_get_all_chapters(raw_chapters)) + + chapters.sort(key=lambda x: x["nr"]) + + return info, chapters diff --git a/sites/provider.py b/sites/provider.py index d91da43..9f3f0dd 100644 --- a/sites/provider.py +++ b/sites/provider.py @@ -3,10 +3,8 @@ from abc import ABC, abstractmethod class Provider(ABC): - @staticmethod - @abstractmethod - def domain() -> str: - pass + domain: str + headers: dict @abstractmethod def get_mediainfo(self) -> tuple[dict, list]: