dojinvoice_db.dlsite

Request-efficient access layer over dlsite_async.

Three kinds of request are used, in increasing cost per work:

  1. Search listing (DlsiteClient.fetch_listing_ids()) — one request returns up to 100 product IDs, newest first.
  2. Product info ajax (DlsiteClient.fetch_product_info()) — DLsite's product/info/ajax endpoint accepts a comma-separated list of product IDs, so one request covers up to 100 works. It yields everything DlsiteAPI.product_info() yields (title, maker, work type, release date, cover image) plus the price/sales/rating figures.
  3. Work page HTML (DlsiteClient.fetch_detail()) — one request per work, the only source for circle name, description, genres and the voice actor / scenario / illustration / music credits.

Only step 3 scales with the number of works, so the crawler performs it once per work and never again unless explicitly asked to.

  1"""Request-efficient access layer over :mod:`dlsite_async`.
  2
  3Three kinds of request are used, in increasing cost per work:
  4
  51. **Search listing** (:meth:`DlsiteClient.fetch_listing_ids`) — one request
  6   returns up to 100 product IDs, newest first.
  72. **Product info ajax** (:meth:`DlsiteClient.fetch_product_info`) — DLsite's
  8   ``product/info/ajax`` endpoint accepts a comma-separated list of product
  9   IDs, so one request covers up to 100 works. It yields everything
 10   ``DlsiteAPI.product_info()`` yields (title, maker, work type, release date,
 11   cover image) *plus* the price/sales/rating figures.
 123. **Work page HTML** (:meth:`DlsiteClient.fetch_detail`) — one request per
 13   work, the only source for circle name, description, genres and the
 14   voice actor / scenario / illustration / music credits.
 15
 16Only step 3 scales with the number of works, so the crawler performs it once
 17per work and never again unless explicitly asked to.
 18"""
 19
 20from __future__ import annotations
 21
 22import asyncio
 23import importlib
 24import json
 25import logging
 26import re
 27import time
 28from collections.abc import Callable
 29from dataclasses import replace
 30from datetime import datetime
 31from typing import TYPE_CHECKING, Any, Self
 32
 33from aiohttp import ClientError, ClientResponseError, ClientTimeout
 34from dlsite_async import AgeCategory, BookType, DlsiteAPI, Work, WorkType
 35
 36if TYPE_CHECKING:
 37    from collections.abc import Callable, Iterable, Mapping, Sequence
 38    from types import TracebackType
 39
 40    from .models import SearchQuery
 41
 42LOGGER = logging.getLogger(__name__)
 43
 44USER_AGENT = "dojinvoice-db (+https://github.com/eggplants/dojinvoice_db)"
 45
 46INFO_URL = "https://www.dlsite.com/maniax/product/info/ajax"
 47
 48_PRODUCT_ID_RE = re.compile(r'data-list_item_product_id="([A-Z]{2}\d+)"')
 49
 50_RETRY_STATUSES = frozenset({408, 425, 429, 500, 502, 503, 504})
 51
 52# A "<" that does not open a tag, a closing tag, a comment/doctype or a PI.
 53# Descriptions written by circles regularly contain bare "<track list>" style
 54# text; BeautifulSoup turns those into elements whose name lxml then rejects.
 55_STRAY_BRACKET_RE = re.compile(r"<(?![!?]|/?[A-Za-z][A-Za-z0-9._:-]*[\s/>])")
 56
 57
 58def _import_parse_work_html() -> Callable[[str], dict[str, Any]] | None:
 59    """Return ``dlsite-async``'s work page parser, or ``None`` if it moved.
 60
 61    It lives in a private module, so the import is guarded: if a future
 62    release restructures it, :meth:`DlsiteClient.fetch_detail` falls back to
 63    the public (but twice as expensive) ``DlsiteAPI.get_work``.
 64    """
 65    try:
 66        module = importlib.import_module("dlsite_async._scraper")
 67    except ImportError:  # pragma: no cover - only if dlsite-async restructures
 68        return None
 69    return getattr(module, "parse_work_html", None)
 70
 71
 72parse_work_html = _import_parse_work_html()
 73
 74
 75class WorkNotFoundError(Exception):
 76    """A work page could not be retrieved (removed, private or wrong ID)."""
 77
 78
 79def extract_product_ids(html: str) -> list[str]:
 80    """Return the product IDs listed on a search results page, in page order.
 81
 82    Duplicates are dropped while preserving the first occurrence.
 83    """
 84    return list(dict.fromkeys(match.group(1) for match in _PRODUCT_ID_RE.finditer(html)))
 85
 86
 87def escape_stray_brackets(html: str) -> str:
 88    """Escape ``<`` characters that do not open a tag.
 89
 90    Work descriptions contain unescaped angle brackets often enough to matter
 91    (``<track list, ...>`` headings), and a bracket followed by Japanese text
 92    becomes a tag name lxml refuses to build an element for.
 93    """
 94    return _STRAY_BRACKET_RE.sub("&lt;", html)
 95
 96
 97def parse_detail_html(html: str) -> dict[str, Any]:
 98    """Parse a work page into the fields ``dlsite_async`` scrapes from it.
 99
100    Retries once with :func:`escape_stray_brackets` when the raw HTML produces
101    an element name the parser rejects, so a single bad description does not
102    cost the work its detail row.
103    """
104    if parse_work_html is None:  # pragma: no cover - dependency fallback
105        msg = "dlsite_async._scraper.parse_work_html is unavailable"
106        raise WorkNotFoundError(msg)
107    try:
108        return parse_work_html(html)
109    except ValueError as e:
110        LOGGER.debug("retrying work page parse with escaped brackets: %s", e)
111        return parse_work_html(escape_stray_brackets(html))
112
113
114def _to_datetime(value: Any) -> datetime | None:  # noqa: ANN401 - raw JSON value
115    if not isinstance(value, str):
116        return None
117    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d"):
118        try:
119            return datetime.strptime(value, fmt)  # noqa: DTZ007 - DLsite times are naive JST
120        except ValueError:
121            continue
122    return None
123
124
125def work_from_info(product_id: str, info: Mapping[str, Any]) -> Work:
126    """Build a :class:`~dlsite_async.Work` from one ajax info entry.
127
128    Unlike ``DlsiteAPI.product_info()`` this never raises on unfamiliar enum
129    values -- DLsite occasionally introduces new work/book types, and dropping
130    the field is better than dropping the work.
131    """
132    data = dict(info)
133    data["product_id"] = product_id
134    data["age_category"] = _enum_or_none(AgeCategory, data.get("age_category"))
135    data["work_type"] = _enum_or_none(WorkType, data.get("work_type"))
136    book_type = data.get("book_type")
137    if isinstance(book_type, dict):
138        book_type = book_type.get("value")
139    data["book_type"] = _enum_or_none(BookType, book_type)
140    data["regist_date"] = _to_datetime(data.get("regist_date"))
141    return Work.from_dict(data)
142
143
144def _enum_or_none(enum_cls: Any, value: Any) -> Any:  # noqa: ANN401 - generic enum helper
145    if value is None:
146        return None
147    try:
148        return enum_cls(value)
149    except ValueError:
150        LOGGER.debug("unknown %s value: %r", enum_cls.__name__, value)
151        return None
152
153
154class _Throttle:
155    """Serialize requests so that at least ``interval`` seconds pass between them."""
156
157    def __init__(self, interval: float) -> None:
158        self._interval = max(interval, 0.0)
159        self._lock = asyncio.Lock()
160        self._next = 0.0
161
162    async def wait(self) -> None:
163        """Block until the next request is allowed to start."""
164        if not self._interval:
165            return
166        async with self._lock:
167            now = time.monotonic()
168            delay = self._next - now
169            if delay > 0:
170                await asyncio.sleep(delay)
171                now = time.monotonic()
172            self._next = now + self._interval
173
174
175class DlsiteClient:
176    """An HTTP session against DLsite with throttling, retries and counters.
177
178    Args:
179        locale: Optional DLsite locale (e.g. ``en_US``). Defaults to Japanese.
180        interval: Minimum seconds between requests.
181        concurrency: Maximum number of in-flight requests.
182        retries: Attempts per request on transient failures.
183        timeout: Per-request timeout in seconds.
184    """
185
186    def __init__(
187        self,
188        *,
189        locale: str | None = None,
190        interval: float = 0.5,
191        concurrency: int = 4,
192        retries: int = 3,
193        timeout: float = 30.0,
194    ) -> None:
195        self.api = DlsiteAPI(locale=locale)
196        self._throttle = _Throttle(interval)
197        self._semaphore = asyncio.Semaphore(max(concurrency, 1))
198        self._retries = max(retries, 1)
199        self._timeout = ClientTimeout(total=timeout)
200        self.request_count = 0
201
202    async def __aenter__(self) -> Self:
203        """Enter the client session."""
204        return self
205
206    async def __aexit__(
207        self,
208        exc_type: type[BaseException] | None,
209        exc: BaseException | None,
210        tb: TracebackType | None,
211    ) -> None:
212        """Close the client session."""
213        await self.close()
214
215    async def close(self) -> None:
216        """Close the underlying HTTP session."""
217        await self.api.close()
218
219    async def _get_text(self, url: str, *, params: Mapping[str, str] | None = None) -> str | None:
220        """GET ``url`` and return its body, or ``None`` on a 404/403.
221
222        Retries transient failures with exponential backoff.
223        """
224        last_error: Exception | None = None
225        for attempt in range(self._retries):
226            async with self._semaphore:
227                await self._throttle.wait()
228                self.request_count += 1
229                try:
230                    async with self.api.get(
231                        url,
232                        params=dict(params or {}),
233                        headers={"User-Agent": USER_AGENT},
234                        timeout=self._timeout,
235                    ) as response:
236                        return await response.text()
237                except ClientResponseError as e:
238                    if e.status not in _RETRY_STATUSES:
239                        if e.status in {403, 404}:
240                            return None
241                        raise
242                    last_error = e
243                except (ClientError, TimeoutError) as e:
244                    last_error = e
245            backoff = 2.0**attempt
246            LOGGER.warning("request failed (%s), retrying in %.0fs: %s", last_error, backoff, url)
247            await asyncio.sleep(backoff)
248        msg = f"giving up on {url}: {last_error}"
249        raise WorkNotFoundError(msg) from last_error
250
251    async def fetch_listing_ids(self, query: SearchQuery, page: int) -> list[str]:
252        """Return the product IDs on ``page`` of the ``query`` listing."""
253        html = await self._get_text(query.page_url(page))
254        if html is None:
255            return []
256        return extract_product_ids(html)
257
258    async def fetch_product_info(self, product_ids: Sequence[str]) -> dict[str, dict[str, Any]]:
259        """Return raw ajax info for up to ~100 product IDs in a single request.
260
261        IDs DLsite does not know about are simply absent from the result.
262        """
263        if not product_ids:
264            return {}
265        text = await self._get_text(INFO_URL, params={"product_id": ",".join(product_ids)})
266        if not text:
267            return {}
268        try:
269            data = json.loads(text)
270        except ValueError:
271            LOGGER.warning("malformed info response for %d ids", len(product_ids))
272            return {}
273        if not isinstance(data, dict):
274            return {}
275        return {k: v for k, v in data.items() if isinstance(v, dict)}
276
277    async def fetch_detail(self, work: Work) -> Work:
278        """Fill in the work-page-only fields of ``work``.
279
280        Raises:
281            WorkNotFoundError: Neither the work nor the announce page exists.
282        """
283        if parse_work_html is None:  # pragma: no cover - dependency fallback
284            return await self.api.get_work(work.product_id)
285        site_id = work.site_id or "maniax"
286        for kind in ("work", "announce"):
287            url = f"https://www.dlsite.com/{site_id}/{kind}/=/product_id/{work.product_id}.html/"
288            html = await self._get_text(url)
289            if html:
290                return replace(work, **parse_detail_html(html))
291        msg = f"no work or announce page for {work.product_id}"
292        raise WorkNotFoundError(msg)
293
294
295def batched(items: Sequence[str], size: int) -> Iterable[Sequence[str]]:
296    """Split ``items`` into consecutive chunks of at most ``size``."""
297    step = max(size, 1)
298    for start in range(0, len(items), step):
299        yield items[start : start + step]
LOGGER = <Logger dojinvoice_db.dlsite (WARNING)>
USER_AGENT = 'dojinvoice-db (+https://github.com/eggplants/dojinvoice_db)'
INFO_URL = 'https://wwwdojinvoice_db.dlsite.com/maniax/product/info/ajax'
def parse_work_html(content: str) -> dict[str, typing.Any]:
155def parse_work_html(content: str) -> dict[str, Any]:
156    """Parse work HTML."""
157    tree = soupparser.fromstring(_clean_xml(content))
158    info: dict[str, Any] = {}
159    for table in (
160        '//table[@id="work_maker"]//tr',
161        '//table[@id="work_outline"]//tr',
162    ):
163        info.update(_parse_work_outline_rows(cast(html.HtmlElement, tree.xpath(table))))
164    for dl in tree.xpath('//dl[@class="c-productInfo__box"]'):
165        info.update(_parse_comipo_product_info(cast(html.HtmlElement, dl)))
166    for div in ('//div[@class="product-slider-data"]//div',):
167        (info.update(_parse_work_slider_data(cast(html.HtmlElement, tree.xpath(div)))),)
168    for meta in tree.xpath('//meta[@name="description"]'):
169        info.update(_parse_work_description(cast(html.HtmlElement, meta)))
170    return info

Parse work HTML.

class WorkNotFoundError(builtins.Exception):
76class WorkNotFoundError(Exception):
77    """A work page could not be retrieved (removed, private or wrong ID)."""

A work page could not be retrieved (removed, private or wrong ID).

def extract_product_ids(html: str) -> list[str]:
80def extract_product_ids(html: str) -> list[str]:
81    """Return the product IDs listed on a search results page, in page order.
82
83    Duplicates are dropped while preserving the first occurrence.
84    """
85    return list(dict.fromkeys(match.group(1) for match in _PRODUCT_ID_RE.finditer(html)))

Return the product IDs listed on a search results page, in page order.

Duplicates are dropped while preserving the first occurrence.

def escape_stray_brackets(html: str) -> str:
88def escape_stray_brackets(html: str) -> str:
89    """Escape ``<`` characters that do not open a tag.
90
91    Work descriptions contain unescaped angle brackets often enough to matter
92    (``<track list, ...>`` headings), and a bracket followed by Japanese text
93    becomes a tag name lxml refuses to build an element for.
94    """
95    return _STRAY_BRACKET_RE.sub("&lt;", html)

Escape < characters that do not open a tag.

Work descriptions contain unescaped angle brackets often enough to matter (<track list, ...> headings), and a bracket followed by Japanese text becomes a tag name lxml refuses to build an element for.

def parse_detail_html(html: str) -> dict[str, typing.Any]:
 98def parse_detail_html(html: str) -> dict[str, Any]:
 99    """Parse a work page into the fields ``dlsite_async`` scrapes from it.
100
101    Retries once with :func:`escape_stray_brackets` when the raw HTML produces
102    an element name the parser rejects, so a single bad description does not
103    cost the work its detail row.
104    """
105    if parse_work_html is None:  # pragma: no cover - dependency fallback
106        msg = "dlsite_async._scraper.parse_work_html is unavailable"
107        raise WorkNotFoundError(msg)
108    try:
109        return parse_work_html(html)
110    except ValueError as e:
111        LOGGER.debug("retrying work page parse with escaped brackets: %s", e)
112        return parse_work_html(escape_stray_brackets(html))

Parse a work page into the fields dlsite_async scrapes from it.

Retries once with escape_stray_brackets() when the raw HTML produces an element name the parser rejects, so a single bad description does not cost the work its detail row.

def work_from_info( product_id: str, info: Mapping[str, typing.Any]) -> dlsite_async.work.Work:
126def work_from_info(product_id: str, info: Mapping[str, Any]) -> Work:
127    """Build a :class:`~dlsite_async.Work` from one ajax info entry.
128
129    Unlike ``DlsiteAPI.product_info()`` this never raises on unfamiliar enum
130    values -- DLsite occasionally introduces new work/book types, and dropping
131    the field is better than dropping the work.
132    """
133    data = dict(info)
134    data["product_id"] = product_id
135    data["age_category"] = _enum_or_none(AgeCategory, data.get("age_category"))
136    data["work_type"] = _enum_or_none(WorkType, data.get("work_type"))
137    book_type = data.get("book_type")
138    if isinstance(book_type, dict):
139        book_type = book_type.get("value")
140    data["book_type"] = _enum_or_none(BookType, book_type)
141    data["regist_date"] = _to_datetime(data.get("regist_date"))
142    return Work.from_dict(data)

Build a ~dlsite_async.Work from one ajax info entry.

Unlike DlsiteAPI.product_info() this never raises on unfamiliar enum values -- DLsite occasionally introduces new work/book types, and dropping the field is better than dropping the work.

class DlsiteClient:
176class DlsiteClient:
177    """An HTTP session against DLsite with throttling, retries and counters.
178
179    Args:
180        locale: Optional DLsite locale (e.g. ``en_US``). Defaults to Japanese.
181        interval: Minimum seconds between requests.
182        concurrency: Maximum number of in-flight requests.
183        retries: Attempts per request on transient failures.
184        timeout: Per-request timeout in seconds.
185    """
186
187    def __init__(
188        self,
189        *,
190        locale: str | None = None,
191        interval: float = 0.5,
192        concurrency: int = 4,
193        retries: int = 3,
194        timeout: float = 30.0,
195    ) -> None:
196        self.api = DlsiteAPI(locale=locale)
197        self._throttle = _Throttle(interval)
198        self._semaphore = asyncio.Semaphore(max(concurrency, 1))
199        self._retries = max(retries, 1)
200        self._timeout = ClientTimeout(total=timeout)
201        self.request_count = 0
202
203    async def __aenter__(self) -> Self:
204        """Enter the client session."""
205        return self
206
207    async def __aexit__(
208        self,
209        exc_type: type[BaseException] | None,
210        exc: BaseException | None,
211        tb: TracebackType | None,
212    ) -> None:
213        """Close the client session."""
214        await self.close()
215
216    async def close(self) -> None:
217        """Close the underlying HTTP session."""
218        await self.api.close()
219
220    async def _get_text(self, url: str, *, params: Mapping[str, str] | None = None) -> str | None:
221        """GET ``url`` and return its body, or ``None`` on a 404/403.
222
223        Retries transient failures with exponential backoff.
224        """
225        last_error: Exception | None = None
226        for attempt in range(self._retries):
227            async with self._semaphore:
228                await self._throttle.wait()
229                self.request_count += 1
230                try:
231                    async with self.api.get(
232                        url,
233                        params=dict(params or {}),
234                        headers={"User-Agent": USER_AGENT},
235                        timeout=self._timeout,
236                    ) as response:
237                        return await response.text()
238                except ClientResponseError as e:
239                    if e.status not in _RETRY_STATUSES:
240                        if e.status in {403, 404}:
241                            return None
242                        raise
243                    last_error = e
244                except (ClientError, TimeoutError) as e:
245                    last_error = e
246            backoff = 2.0**attempt
247            LOGGER.warning("request failed (%s), retrying in %.0fs: %s", last_error, backoff, url)
248            await asyncio.sleep(backoff)
249        msg = f"giving up on {url}: {last_error}"
250        raise WorkNotFoundError(msg) from last_error
251
252    async def fetch_listing_ids(self, query: SearchQuery, page: int) -> list[str]:
253        """Return the product IDs on ``page`` of the ``query`` listing."""
254        html = await self._get_text(query.page_url(page))
255        if html is None:
256            return []
257        return extract_product_ids(html)
258
259    async def fetch_product_info(self, product_ids: Sequence[str]) -> dict[str, dict[str, Any]]:
260        """Return raw ajax info for up to ~100 product IDs in a single request.
261
262        IDs DLsite does not know about are simply absent from the result.
263        """
264        if not product_ids:
265            return {}
266        text = await self._get_text(INFO_URL, params={"product_id": ",".join(product_ids)})
267        if not text:
268            return {}
269        try:
270            data = json.loads(text)
271        except ValueError:
272            LOGGER.warning("malformed info response for %d ids", len(product_ids))
273            return {}
274        if not isinstance(data, dict):
275            return {}
276        return {k: v for k, v in data.items() if isinstance(v, dict)}
277
278    async def fetch_detail(self, work: Work) -> Work:
279        """Fill in the work-page-only fields of ``work``.
280
281        Raises:
282            WorkNotFoundError: Neither the work nor the announce page exists.
283        """
284        if parse_work_html is None:  # pragma: no cover - dependency fallback
285            return await self.api.get_work(work.product_id)
286        site_id = work.site_id or "maniax"
287        for kind in ("work", "announce"):
288            url = f"https://www.dlsite.com/{site_id}/{kind}/=/product_id/{work.product_id}.html/"
289            html = await self._get_text(url)
290            if html:
291                return replace(work, **parse_detail_html(html))
292        msg = f"no work or announce page for {work.product_id}"
293        raise WorkNotFoundError(msg)

An HTTP session against DLsite with throttling, retries and counters.

Arguments:
  • locale: Optional DLsite locale (e.g. en_US). Defaults to Japanese.
  • interval: Minimum seconds between requests.
  • concurrency: Maximum number of in-flight requests.
  • retries: Attempts per request on transient failures.
  • timeout: Per-request timeout in seconds.
DlsiteClient( *, locale: str | None = None, interval: float = 0.5, concurrency: int = 4, retries: int = 3, timeout: float = 30.0)
187    def __init__(
188        self,
189        *,
190        locale: str | None = None,
191        interval: float = 0.5,
192        concurrency: int = 4,
193        retries: int = 3,
194        timeout: float = 30.0,
195    ) -> None:
196        self.api = DlsiteAPI(locale=locale)
197        self._throttle = _Throttle(interval)
198        self._semaphore = asyncio.Semaphore(max(concurrency, 1))
199        self._retries = max(retries, 1)
200        self._timeout = ClientTimeout(total=timeout)
201        self.request_count = 0
api
request_count
async def close(self) -> None:
216    async def close(self) -> None:
217        """Close the underlying HTTP session."""
218        await self.api.close()

Close the underlying HTTP session.

async def fetch_listing_ids(self, query: dojinvoice_db.models.SearchQuery, page: int) -> list[str]:
252    async def fetch_listing_ids(self, query: SearchQuery, page: int) -> list[str]:
253        """Return the product IDs on ``page`` of the ``query`` listing."""
254        html = await self._get_text(query.page_url(page))
255        if html is None:
256            return []
257        return extract_product_ids(html)

Return the product IDs on page of the query listing.

async def fetch_product_info(self, product_ids: Sequence[str]) -> dict[str, dict[str, typing.Any]]:
259    async def fetch_product_info(self, product_ids: Sequence[str]) -> dict[str, dict[str, Any]]:
260        """Return raw ajax info for up to ~100 product IDs in a single request.
261
262        IDs DLsite does not know about are simply absent from the result.
263        """
264        if not product_ids:
265            return {}
266        text = await self._get_text(INFO_URL, params={"product_id": ",".join(product_ids)})
267        if not text:
268            return {}
269        try:
270            data = json.loads(text)
271        except ValueError:
272            LOGGER.warning("malformed info response for %d ids", len(product_ids))
273            return {}
274        if not isinstance(data, dict):
275            return {}
276        return {k: v for k, v in data.items() if isinstance(v, dict)}

Return raw ajax info for up to ~100 product IDs in a single request.

IDs DLsite does not know about are simply absent from the result.

async def fetch_detail(self, work: dlsite_async.work.Work) -> dlsite_async.work.Work:
278    async def fetch_detail(self, work: Work) -> Work:
279        """Fill in the work-page-only fields of ``work``.
280
281        Raises:
282            WorkNotFoundError: Neither the work nor the announce page exists.
283        """
284        if parse_work_html is None:  # pragma: no cover - dependency fallback
285            return await self.api.get_work(work.product_id)
286        site_id = work.site_id or "maniax"
287        for kind in ("work", "announce"):
288            url = f"https://www.dlsite.com/{site_id}/{kind}/=/product_id/{work.product_id}.html/"
289            html = await self._get_text(url)
290            if html:
291                return replace(work, **parse_detail_html(html))
292        msg = f"no work or announce page for {work.product_id}"
293        raise WorkNotFoundError(msg)

Fill in the work-page-only fields of work.

Raises:
  • WorkNotFoundError: Neither the work nor the announce page exists.
def batched(items: Sequence[str], size: int) -> Iterable[Sequence[str]]:
296def batched(items: Sequence[str], size: int) -> Iterable[Sequence[str]]:
297    """Split ``items`` into consecutive chunks of at most ``size``."""
298    step = max(size, 1)
299    for start in range(0, len(items), step):
300        yield items[start : start + step]

Split items into consecutive chunks of at most size.