Added MangaPill. Updated to Python 3.14. Improved download performace through async improvments

2026-03-12 14:18:32 +01:00
parent 9cd15e33fb
commit c6c4396f55
7 changed files with 248 additions and 78 deletions
@@ -1,19 +1,29 @@
+import asyncio
 import tempfile
 import xml.etree.ElementTree as XML
 from pathlib import Path

+import aiofiles
+import aiohttp
 import questionary
-from packager import download_images, zip_files
-from rich import print
+from aiohttp import ClientError
+from rich import print, status
 from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn, TextColumn
+
+from packager import zip_files
 from sites import Provider


 class Manga:
-    def __init__(self, provider: Provider, cached: list[int] = []) -> None:
+    def __init__(self, provider: Provider, cached: list[float] = []) -> None:
        self.site = provider
        self.cached = cached
+
+        spinner = status.Status("Fetching manga info")
+        spinner.start()
        self.info, self.chapters = self.site.get_mediainfo()
+        spinner.stop()
+
        self.slug = self.slugify(self.info.get("series"))  # type: ignore

    @staticmethod
@@ -26,9 +36,9 @@ class Manga:
        ComicInfo = XML.Element("ComicInfo")

        XML.SubElement(ComicInfo, "Series").text = self.info.get("series")
-        XML.SubElement(ComicInfo, "Writer").text = ",".join(self.info.get("writer"))  # type: ignore
-        XML.SubElement(ComicInfo, "Penciller").text = ",".join(self.info.get("penciller"))  # type: ignore
-        XML.SubElement(ComicInfo, "Genre").text = ",".join(self.info.get("genre"))  # type: ignore
+        XML.SubElement(ComicInfo, "Writer").text = ",".join(self.info.get("writer", []))  # type: ignore
+        XML.SubElement(ComicInfo, "Penciller").text = ",".join(self.info.get("penciller", []))  # type: ignore
+        XML.SubElement(ComicInfo, "Genre").text = ",".join(self.info.get("genre", []))  # type: ignore
        XML.SubElement(ComicInfo, "Summary").text = self.info.get("summary")
        XML.SubElement(ComicInfo, "Number").text = str(chapter.get("nr"))
        XML.SubElement(ComicInfo, "Title").text = chapter.get("title")
@@ -52,7 +62,7 @@ class Manga:
    def choose_chapters(self, auto):
        choices = [
            questionary.Choice(
-                f"{chapter.get('nr'):02d}: {chapter.get('title')}",
+                f"{chapter.get('nr'):g}: {chapter.get('title')}",
                chapter,
                checked=chapter.get("nr") not in self.cached,
            )
@@ -72,8 +82,19 @@ class Manga:
        )

    def download(self, save_dir: Path):
-        with tempfile.TemporaryDirectory(prefix="manga_") as temp_dir:
-            if self.chapters:
+        if not self.chapters:
+            print("No new [orange3]Chapter[/] were found...")
+            return
+
+        async def process_all_chapters():
+
+            chapter_sem = asyncio.Semaphore(8)
+            image_sem = asyncio.Semaphore(40)
+
+            connector = aiohttp.TCPConnector(limit=100, ttl_dns_cache=300)
+            async with aiohttp.ClientSession(
+                headers=self.site.headers, connector=connector
+            ) as session:
                with Progress(
                    TextColumn("[progress.description]{task.description}"),
                    MofNCompleteColumn(),
@@ -83,15 +104,71 @@ class Manga:
                        f"[bold]Processing [orange3]{self.info.get('series')}[/]",
                        total=len(self.chapters),
                    )
-                    while not p.finished:
-                        for chapter in self.chapters:
-                            print(
-                                f"Downloading [bold blue]Chapter {chapter.get('nr'):02d}[/]..."
+
+                    async def download_and_zip(chapter):
+                        async with chapter_sem:
+                            chapter_nr = chapter.get("nr")
+                            chapter_temp = Path(temp_dir) / f"ch_{chapter_nr:g}"
+                            chapter_temp.mkdir(parents=True, exist_ok=True)
+                            info_file = self._generateComicInfo(chapter, chapter_temp)
+                            img_urls = chapter.get("images", [])
+                            img_files = await self._async_download_images_internal(
+                                img_urls, chapter_temp, session, image_sem
                            )
-                            files = [self._generateComicInfo(chapter, Path(temp_dir))]
-                            files.extend(download_images(chapter.get("images"), Path(temp_dir)))  # type: ignore
-                            zip_files(self.slug, chapter, files, save_dir)
-                            self.cached.append(chapter.get("nr"))
+
+                            all_files = [info_file] + [f for f in img_files if f]
+                            zip_files(self.slug, chapter, all_files, save_dir)
+
+                            self.cached.append(chapter_nr)
                            p.update(task, advance=1)
-            else:
-                print("No new [orange3]Chapter[/] were found...")
+
+                    await asyncio.gather(
+                        *(download_and_zip(ch) for ch in self.chapters)
+                    )
+
+        with tempfile.TemporaryDirectory(prefix="manga_") as temp_dir:
+            asyncio.run(process_all_chapters())
+
+    async def _async_download_images_internal(self, urls, temp_dir, session, semaphore):
+        """Worker to manage the image download tasks for a single chapter."""
+
+        async def bounded_download(url):
+            async with semaphore:
+                return await self.download_image(
+                    session, url, temp_dir, self.site.headers
+                )
+
+        tasks = [bounded_download(url) for url in urls]
+        return await asyncio.gather(*tasks)
+
+    @staticmethod
+    async def download_image(
+        session: aiohttp.ClientSession,
+        url: str,
+        temp_dir: Path,
+        headers: dict,
+        retries=3,
+        timeout=10,
+    ):
+
+        filename = temp_dir.joinpath(Path(url).name)
+
+        for attempt in range(1, retries + 1):
+            try:
+                async with session.get(url, headers=headers, timeout=timeout) as response:  # type: ignore
+                    if response.status == 200:
+                        async with aiofiles.open(filename, "wb") as f:
+                            await f.write(await response.read())
+                        return filename
+                    else:
+                        raise ClientError(f"Bad status {response.status} for {url}")
+            except (
+                asyncio.TimeoutError,
+                ClientError,
+                aiohttp.ClientConnectorError,
+            ) as e:
+                if attempt < retries:
+                    await asyncio.sleep(2**attempt)
+                else:
+                    print(f"Failed to download {url}: {e}")
+                    return None
@@ -1,55 +1,10 @@
-import asyncio
 import zipfile
 from pathlib import Path

-import aiofiles
-import aiohttp
-from aiohttp import ClientError
-
-
-async def download_image(
-    session: aiohttp.ClientSession, url: str, temp_dir: Path, retries=3, timeout=10
-):
-
-    filename = temp_dir.joinpath(Path(url).name)
-
-    for attempt in range(1, retries + 1):
-        try:
-            async with session.get(url, timeout=timeout) as response:  # type: ignore
-                if response.status == 200:
-                    async with aiofiles.open(filename, "wb") as f:
-                        await f.write(await response.read())
-                    return filename
-                else:
-                    raise ClientError(f"Bad status {response.status} for {url}")
-        except (asyncio.TimeoutError, ClientError, aiohttp.ClientConnectorError) as e:
-            if attempt < retries:
-                await asyncio.sleep(2**attempt)
-            else:
-                print(f"Failed to download {url}: {e}")
-                return None
-
-
-async def _download_images(urls, temp_dir: Path, concurrency=5):
-    semaphore = asyncio.Semaphore(concurrency)
-
-    async with aiohttp.ClientSession() as session:
-
-        async def bounded_download(url):
-            async with semaphore:
-                return await download_image(session, url, temp_dir)
-
-        tasks = [bounded_download(url) for url in urls]
-        return await asyncio.gather(*tasks)
-
-
-def download_images(images: list[str], temp_dir: Path):
-    return asyncio.run(_download_images(images, temp_dir))
-

 def zip_files(slug, chapter: dict, files: list[Path], save_dir: Path):
    save_dir.joinpath(slug).mkdir(exist_ok=True)
-    filename = save_dir.joinpath(slug).joinpath(f"{slug} - Ch. {chapter.get('nr')}.cbz")  # type: ignore
+    filename = save_dir.joinpath(slug).joinpath(f"{slug} - Ch. {chapter.get('nr'):g}.cbz")  # type: ignore
    with zipfile.ZipFile(filename, "w") as zf:
        for file in files:
            zf.write(file, file.name)
@@ -1,12 +1,11 @@
 from .flamecomics import FlameComics
+from .mangapill import Mangapill
 from .provider import Provider

-PROVIDERS = [
-    FlameComics,
-]
+PROVIDERS = [FlameComics, Mangapill]


 def parse_provider(session, url) -> Provider | None:
    for site in PROVIDERS:
-        if site.domain() in url:
+        if site.domain in url:
            return site(session, url)
@@ -15,6 +15,15 @@ CHAPTER_URL = "https://flamecomics.xyz/series/{id}/{token}"

 class FlameComics(Provider):

+    domain = "flamecomics.xyz"
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/120.0.0.0 Safari/537.36"
+        )
+    }
+
    def __init__(self, session: requests.Session, url) -> None:
        self.s = session
        self.id = self.parse_url(url)
@@ -50,7 +59,7 @@ class FlameComics(Provider):
    def parse_chapters(self, payload: list[dict]) -> list[dict]:
        chapters = list()
        for c in payload:
-            nbr = int(float(c.get("chapter", 0.0)))
+            nbr = float(c.get("chapter", 0.0))
            title = c.get("title")
            if not title:
                title = f"Chapter {nbr}"
@@ -88,7 +97,3 @@ class FlameComics(Provider):

    def get_url(self) -> str:
        return INFO_URL.format(id=self.id)
-
-    @staticmethod
-    def domain() -> str:
-        return "flamecomics.xyz"
@@ -0,0 +1,136 @@
+import asyncio
+import re
+from typing import TYPE_CHECKING
+
+import aiohttp
+from bs4 import BeautifulSoup as bs
+
+from .provider import Provider
+
+if TYPE_CHECKING:
+    import requests
+
+BASE_URL = "https://mangapill.com"
+INFO_URL = "https://mangapill.com/manga/{id}"
+
+# CDN_URL = "https://cdn.readdetectiveconan.com"
+# IMAGE_URL = CDN_URL + "/file/mangap/{id}/10{chapter_nr:03d}000/{file}"
+# CHAPTER_URL = "https://mangapill.com/chapters/{id}-10{chapter_nr:03d}000"
+
+
+class Mangapill(Provider):
+    domain = "mangapill.com"
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/120.0.0.0 Safari/537.36"
+        ),
+        "Referer": "https://mangapill.com/",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+    }
+
+    def __init__(self, session: "requests.Session", url) -> None:
+        self.s = session
+        self.id = self.parse_url(url)
+
+    def parse_url(self, url: str) -> str:
+        parse = re.search(r"manga\/(?P<id>\d+)", url)
+        return parse.group("id") if parse else ""
+
+    def get_url(self) -> str:
+        return INFO_URL.format(id=self.id)
+
+    @staticmethod
+    def parse_info(payload: bs) -> dict:
+        title_elem = payload.select_one("h1.font-bold")
+        title = title_elem.get_text(strip=True) if title_elem else "Unknown Title"
+        desc_p = payload.select_one("p.text-sm.text--secondary")
+        description = None
+
+        if desc_p:
+            html_content = desc_p.decode_contents()
+            if "<br><br>" in html_content:
+                html_content = html_content.split("<br><br>", 1)[1]
+            clean_soup = bs(html_content, "html.parser")
+            text = clean_soup.get_text("\n", strip=True)
+            lines = [line.strip() for line in text.split("\n") if line.strip()]
+            for i, line in enumerate(lines):
+                if line.startswith("The "):
+                    lines = lines[i:]
+                    break
+            description = "\n\n".join(lines) if lines else None
+
+        genres = [
+            a.get_text(strip=True) for a in payload.select("a[href^='/search?genre=']")
+        ]
+
+        return {
+            "series": title,
+            "genre": genres,
+            "summary": description,
+            "scanInformation": "MangaPill",
+        }
+
+    async def _fetch_chapter(
+        self,
+        session: aiohttp.ClientSession,
+        href: str,
+        title: str,
+        nbr: float,
+        semaphore: asyncio.Semaphore,
+    ):
+        """Internal async worker to fetch a single chapter's images."""
+        async with semaphore:
+            try:
+                async with session.get(BASE_URL + href) as response:
+                    content = await response.read()
+                    soup = bs(content, "html.parser")
+
+                    images = []
+                    for img in soup.select("img.js-page"):
+                        src = img.get("data-src") or img.get("src")
+                        if src:
+                            images.append(src)
+
+                    return {"nr": nbr, "title": title, "images": images}
+            except Exception:
+                return {"nr": nbr, "title": title, "images": []}
+
+    async def _async_get_all_chapters(self, chapter_tags: list) -> list:
+        """Sets up the concurrent tasks and executes them."""
+        semaphore = asyncio.Semaphore(25)  # Limits to 15 concurrent requests
+        total_ch = len(chapter_tags)
+
+        async with aiohttp.ClientSession(headers=self.headers) as session:
+            tasks = []
+            for i, a in enumerate(chapter_tags):
+                chapter_title = a.get_text(strip=True)
+                match = re.search(
+                    r"Chapter\s*(?P<nbr>[\d.]+)", chapter_title, re.IGNORECASE
+                )
+                nbr = float(match.group("nbr")) if match else float(total_ch - i)
+
+                tasks.append(
+                    self._fetch_chapter(
+                        session, a["href"], chapter_title, nbr, semaphore
+                    )
+                )
+
+            return await asyncio.gather(*tasks)
+
+    def get_mediainfo(self) -> tuple[dict, list]:
+
+        r = self.s.get(self.get_url())
+        series_page = bs(r.content, "html.parser")
+
+        info = self.parse_info(series_page)
+
+        raw_chapters = series_page.select("#chapters a[href^='/chapters/']")
+
+        chapters = asyncio.run(self._async_get_all_chapters(raw_chapters))
+
+        chapters.sort(key=lambda x: x["nr"])
+
+        return info, chapters
@@ -3,10 +3,8 @@ from abc import ABC, abstractmethod

 class Provider(ABC):

-    @staticmethod
-    @abstractmethod
-    def domain() -> str:
-        pass
+    domain: str
+    headers: dict

    @abstractmethod
    def get_mediainfo(self) -> tuple[dict, list]: