dojinvoice_db.dlsite
Request-efficient access layer over dlsite_async.
Three kinds of request are used, in increasing cost per work:
- Search listing (
DlsiteClient.fetch_listing_ids()) — one request returns up to 100 product IDs, newest first. - Product info ajax (
DlsiteClient.fetch_product_info()) — DLsite'sproduct/info/ajaxendpoint accepts a comma-separated list of product IDs, so one request covers up to 100 works. It yields everythingDlsiteAPI.product_info()yields (title, maker, work type, release date, cover image) plus the price/sales/rating figures. - Work page HTML (
DlsiteClient.fetch_detail()) — one request per work, the only source for circle name, description, genres and the voice actor / scenario / illustration / music credits.
Only step 3 scales with the number of works, so the crawler performs it once per work and never again unless explicitly asked to.
1"""Request-efficient access layer over :mod:`dlsite_async`. 2 3Three kinds of request are used, in increasing cost per work: 4 51. **Search listing** (:meth:`DlsiteClient.fetch_listing_ids`) — one request 6 returns up to 100 product IDs, newest first. 72. **Product info ajax** (:meth:`DlsiteClient.fetch_product_info`) — DLsite's 8 ``product/info/ajax`` endpoint accepts a comma-separated list of product 9 IDs, so one request covers up to 100 works. It yields everything 10 ``DlsiteAPI.product_info()`` yields (title, maker, work type, release date, 11 cover image) *plus* the price/sales/rating figures. 123. **Work page HTML** (:meth:`DlsiteClient.fetch_detail`) — one request per 13 work, the only source for circle name, description, genres and the 14 voice actor / scenario / illustration / music credits. 15 16Only step 3 scales with the number of works, so the crawler performs it once 17per work and never again unless explicitly asked to. 18""" 19 20from __future__ import annotations 21 22import asyncio 23import importlib 24import json 25import logging 26import re 27import time 28from collections.abc import Callable 29from dataclasses import replace 30from datetime import datetime 31from typing import TYPE_CHECKING, Any, Self 32 33from aiohttp import ClientError, ClientResponseError, ClientTimeout 34from dlsite_async import AgeCategory, BookType, DlsiteAPI, Work, WorkType 35 36if TYPE_CHECKING: 37 from collections.abc import Callable, Iterable, Mapping, Sequence 38 from types import TracebackType 39 40 from .models import SearchQuery 41 42LOGGER = logging.getLogger(__name__) 43 44USER_AGENT = "dojinvoice-db (+https://github.com/eggplants/dojinvoice_db)" 45 46INFO_URL = "https://www.dlsite.com/maniax/product/info/ajax" 47 48_PRODUCT_ID_RE = re.compile(r'data-list_item_product_id="([A-Z]{2}\d+)"') 49 50_RETRY_STATUSES = frozenset({408, 425, 429, 500, 502, 503, 504}) 51 52# A "<" that does not open a tag, a closing tag, a comment/doctype or a PI. 53# Descriptions written by circles regularly contain bare "<track list>" style 54# text; BeautifulSoup turns those into elements whose name lxml then rejects. 55_STRAY_BRACKET_RE = re.compile(r"<(?![!?]|/?[A-Za-z][A-Za-z0-9._:-]*[\s/>])") 56 57 58def _import_parse_work_html() -> Callable[[str], dict[str, Any]] | None: 59 """Return ``dlsite-async``'s work page parser, or ``None`` if it moved. 60 61 It lives in a private module, so the import is guarded: if a future 62 release restructures it, :meth:`DlsiteClient.fetch_detail` falls back to 63 the public (but twice as expensive) ``DlsiteAPI.get_work``. 64 """ 65 try: 66 module = importlib.import_module("dlsite_async._scraper") 67 except ImportError: # pragma: no cover - only if dlsite-async restructures 68 return None 69 return getattr(module, "parse_work_html", None) 70 71 72parse_work_html = _import_parse_work_html() 73 74 75class WorkNotFoundError(Exception): 76 """A work page could not be retrieved (removed, private or wrong ID).""" 77 78 79def extract_product_ids(html: str) -> list[str]: 80 """Return the product IDs listed on a search results page, in page order. 81 82 Duplicates are dropped while preserving the first occurrence. 83 """ 84 return list(dict.fromkeys(match.group(1) for match in _PRODUCT_ID_RE.finditer(html))) 85 86 87def escape_stray_brackets(html: str) -> str: 88 """Escape ``<`` characters that do not open a tag. 89 90 Work descriptions contain unescaped angle brackets often enough to matter 91 (``<track list, ...>`` headings), and a bracket followed by Japanese text 92 becomes a tag name lxml refuses to build an element for. 93 """ 94 return _STRAY_BRACKET_RE.sub("<", html) 95 96 97def parse_detail_html(html: str) -> dict[str, Any]: 98 """Parse a work page into the fields ``dlsite_async`` scrapes from it. 99 100 Retries once with :func:`escape_stray_brackets` when the raw HTML produces 101 an element name the parser rejects, so a single bad description does not 102 cost the work its detail row. 103 """ 104 if parse_work_html is None: # pragma: no cover - dependency fallback 105 msg = "dlsite_async._scraper.parse_work_html is unavailable" 106 raise WorkNotFoundError(msg) 107 try: 108 return parse_work_html(html) 109 except ValueError as e: 110 LOGGER.debug("retrying work page parse with escaped brackets: %s", e) 111 return parse_work_html(escape_stray_brackets(html)) 112 113 114def _to_datetime(value: Any) -> datetime | None: # noqa: ANN401 - raw JSON value 115 if not isinstance(value, str): 116 return None 117 for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d"): 118 try: 119 return datetime.strptime(value, fmt) # noqa: DTZ007 - DLsite times are naive JST 120 except ValueError: 121 continue 122 return None 123 124 125def work_from_info(product_id: str, info: Mapping[str, Any]) -> Work: 126 """Build a :class:`~dlsite_async.Work` from one ajax info entry. 127 128 Unlike ``DlsiteAPI.product_info()`` this never raises on unfamiliar enum 129 values -- DLsite occasionally introduces new work/book types, and dropping 130 the field is better than dropping the work. 131 """ 132 data = dict(info) 133 data["product_id"] = product_id 134 data["age_category"] = _enum_or_none(AgeCategory, data.get("age_category")) 135 data["work_type"] = _enum_or_none(WorkType, data.get("work_type")) 136 book_type = data.get("book_type") 137 if isinstance(book_type, dict): 138 book_type = book_type.get("value") 139 data["book_type"] = _enum_or_none(BookType, book_type) 140 data["regist_date"] = _to_datetime(data.get("regist_date")) 141 return Work.from_dict(data) 142 143 144def _enum_or_none(enum_cls: Any, value: Any) -> Any: # noqa: ANN401 - generic enum helper 145 if value is None: 146 return None 147 try: 148 return enum_cls(value) 149 except ValueError: 150 LOGGER.debug("unknown %s value: %r", enum_cls.__name__, value) 151 return None 152 153 154class _Throttle: 155 """Serialize requests so that at least ``interval`` seconds pass between them.""" 156 157 def __init__(self, interval: float) -> None: 158 self._interval = max(interval, 0.0) 159 self._lock = asyncio.Lock() 160 self._next = 0.0 161 162 async def wait(self) -> None: 163 """Block until the next request is allowed to start.""" 164 if not self._interval: 165 return 166 async with self._lock: 167 now = time.monotonic() 168 delay = self._next - now 169 if delay > 0: 170 await asyncio.sleep(delay) 171 now = time.monotonic() 172 self._next = now + self._interval 173 174 175class DlsiteClient: 176 """An HTTP session against DLsite with throttling, retries and counters. 177 178 Args: 179 locale: Optional DLsite locale (e.g. ``en_US``). Defaults to Japanese. 180 interval: Minimum seconds between requests. 181 concurrency: Maximum number of in-flight requests. 182 retries: Attempts per request on transient failures. 183 timeout: Per-request timeout in seconds. 184 """ 185 186 def __init__( 187 self, 188 *, 189 locale: str | None = None, 190 interval: float = 0.5, 191 concurrency: int = 4, 192 retries: int = 3, 193 timeout: float = 30.0, 194 ) -> None: 195 self.api = DlsiteAPI(locale=locale) 196 self._throttle = _Throttle(interval) 197 self._semaphore = asyncio.Semaphore(max(concurrency, 1)) 198 self._retries = max(retries, 1) 199 self._timeout = ClientTimeout(total=timeout) 200 self.request_count = 0 201 202 async def __aenter__(self) -> Self: 203 """Enter the client session.""" 204 return self 205 206 async def __aexit__( 207 self, 208 exc_type: type[BaseException] | None, 209 exc: BaseException | None, 210 tb: TracebackType | None, 211 ) -> None: 212 """Close the client session.""" 213 await self.close() 214 215 async def close(self) -> None: 216 """Close the underlying HTTP session.""" 217 await self.api.close() 218 219 async def _get_text(self, url: str, *, params: Mapping[str, str] | None = None) -> str | None: 220 """GET ``url`` and return its body, or ``None`` on a 404/403. 221 222 Retries transient failures with exponential backoff. 223 """ 224 last_error: Exception | None = None 225 for attempt in range(self._retries): 226 async with self._semaphore: 227 await self._throttle.wait() 228 self.request_count += 1 229 try: 230 async with self.api.get( 231 url, 232 params=dict(params or {}), 233 headers={"User-Agent": USER_AGENT}, 234 timeout=self._timeout, 235 ) as response: 236 return await response.text() 237 except ClientResponseError as e: 238 if e.status not in _RETRY_STATUSES: 239 if e.status in {403, 404}: 240 return None 241 raise 242 last_error = e 243 except (ClientError, TimeoutError) as e: 244 last_error = e 245 backoff = 2.0**attempt 246 LOGGER.warning("request failed (%s), retrying in %.0fs: %s", last_error, backoff, url) 247 await asyncio.sleep(backoff) 248 msg = f"giving up on {url}: {last_error}" 249 raise WorkNotFoundError(msg) from last_error 250 251 async def fetch_listing_ids(self, query: SearchQuery, page: int) -> list[str]: 252 """Return the product IDs on ``page`` of the ``query`` listing.""" 253 html = await self._get_text(query.page_url(page)) 254 if html is None: 255 return [] 256 return extract_product_ids(html) 257 258 async def fetch_product_info(self, product_ids: Sequence[str]) -> dict[str, dict[str, Any]]: 259 """Return raw ajax info for up to ~100 product IDs in a single request. 260 261 IDs DLsite does not know about are simply absent from the result. 262 """ 263 if not product_ids: 264 return {} 265 text = await self._get_text(INFO_URL, params={"product_id": ",".join(product_ids)}) 266 if not text: 267 return {} 268 try: 269 data = json.loads(text) 270 except ValueError: 271 LOGGER.warning("malformed info response for %d ids", len(product_ids)) 272 return {} 273 if not isinstance(data, dict): 274 return {} 275 return {k: v for k, v in data.items() if isinstance(v, dict)} 276 277 async def fetch_detail(self, work: Work) -> Work: 278 """Fill in the work-page-only fields of ``work``. 279 280 Raises: 281 WorkNotFoundError: Neither the work nor the announce page exists. 282 """ 283 if parse_work_html is None: # pragma: no cover - dependency fallback 284 return await self.api.get_work(work.product_id) 285 site_id = work.site_id or "maniax" 286 for kind in ("work", "announce"): 287 url = f"https://www.dlsite.com/{site_id}/{kind}/=/product_id/{work.product_id}.html/" 288 html = await self._get_text(url) 289 if html: 290 return replace(work, **parse_detail_html(html)) 291 msg = f"no work or announce page for {work.product_id}" 292 raise WorkNotFoundError(msg) 293 294 295def batched(items: Sequence[str], size: int) -> Iterable[Sequence[str]]: 296 """Split ``items`` into consecutive chunks of at most ``size``.""" 297 step = max(size, 1) 298 for start in range(0, len(items), step): 299 yield items[start : start + step]
155def parse_work_html(content: str) -> dict[str, Any]: 156 """Parse work HTML.""" 157 tree = soupparser.fromstring(_clean_xml(content)) 158 info: dict[str, Any] = {} 159 for table in ( 160 '//table[@id="work_maker"]//tr', 161 '//table[@id="work_outline"]//tr', 162 ): 163 info.update(_parse_work_outline_rows(cast(html.HtmlElement, tree.xpath(table)))) 164 for dl in tree.xpath('//dl[@class="c-productInfo__box"]'): 165 info.update(_parse_comipo_product_info(cast(html.HtmlElement, dl))) 166 for div in ('//div[@class="product-slider-data"]//div',): 167 (info.update(_parse_work_slider_data(cast(html.HtmlElement, tree.xpath(div)))),) 168 for meta in tree.xpath('//meta[@name="description"]'): 169 info.update(_parse_work_description(cast(html.HtmlElement, meta))) 170 return info
Parse work HTML.
76class WorkNotFoundError(Exception): 77 """A work page could not be retrieved (removed, private or wrong ID)."""
A work page could not be retrieved (removed, private or wrong ID).
80def extract_product_ids(html: str) -> list[str]: 81 """Return the product IDs listed on a search results page, in page order. 82 83 Duplicates are dropped while preserving the first occurrence. 84 """ 85 return list(dict.fromkeys(match.group(1) for match in _PRODUCT_ID_RE.finditer(html)))
Return the product IDs listed on a search results page, in page order.
Duplicates are dropped while preserving the first occurrence.
88def escape_stray_brackets(html: str) -> str: 89 """Escape ``<`` characters that do not open a tag. 90 91 Work descriptions contain unescaped angle brackets often enough to matter 92 (``<track list, ...>`` headings), and a bracket followed by Japanese text 93 becomes a tag name lxml refuses to build an element for. 94 """ 95 return _STRAY_BRACKET_RE.sub("<", html)
Escape < characters that do not open a tag.
Work descriptions contain unescaped angle brackets often enough to matter
(<track list, ...> headings), and a bracket followed by Japanese text
becomes a tag name lxml refuses to build an element for.
98def parse_detail_html(html: str) -> dict[str, Any]: 99 """Parse a work page into the fields ``dlsite_async`` scrapes from it. 100 101 Retries once with :func:`escape_stray_brackets` when the raw HTML produces 102 an element name the parser rejects, so a single bad description does not 103 cost the work its detail row. 104 """ 105 if parse_work_html is None: # pragma: no cover - dependency fallback 106 msg = "dlsite_async._scraper.parse_work_html is unavailable" 107 raise WorkNotFoundError(msg) 108 try: 109 return parse_work_html(html) 110 except ValueError as e: 111 LOGGER.debug("retrying work page parse with escaped brackets: %s", e) 112 return parse_work_html(escape_stray_brackets(html))
Parse a work page into the fields dlsite_async scrapes from it.
Retries once with escape_stray_brackets() when the raw HTML produces
an element name the parser rejects, so a single bad description does not
cost the work its detail row.
126def work_from_info(product_id: str, info: Mapping[str, Any]) -> Work: 127 """Build a :class:`~dlsite_async.Work` from one ajax info entry. 128 129 Unlike ``DlsiteAPI.product_info()`` this never raises on unfamiliar enum 130 values -- DLsite occasionally introduces new work/book types, and dropping 131 the field is better than dropping the work. 132 """ 133 data = dict(info) 134 data["product_id"] = product_id 135 data["age_category"] = _enum_or_none(AgeCategory, data.get("age_category")) 136 data["work_type"] = _enum_or_none(WorkType, data.get("work_type")) 137 book_type = data.get("book_type") 138 if isinstance(book_type, dict): 139 book_type = book_type.get("value") 140 data["book_type"] = _enum_or_none(BookType, book_type) 141 data["regist_date"] = _to_datetime(data.get("regist_date")) 142 return Work.from_dict(data)
Build a ~dlsite_async.Work from one ajax info entry.
Unlike DlsiteAPI.product_info() this never raises on unfamiliar enum
values -- DLsite occasionally introduces new work/book types, and dropping
the field is better than dropping the work.
176class DlsiteClient: 177 """An HTTP session against DLsite with throttling, retries and counters. 178 179 Args: 180 locale: Optional DLsite locale (e.g. ``en_US``). Defaults to Japanese. 181 interval: Minimum seconds between requests. 182 concurrency: Maximum number of in-flight requests. 183 retries: Attempts per request on transient failures. 184 timeout: Per-request timeout in seconds. 185 """ 186 187 def __init__( 188 self, 189 *, 190 locale: str | None = None, 191 interval: float = 0.5, 192 concurrency: int = 4, 193 retries: int = 3, 194 timeout: float = 30.0, 195 ) -> None: 196 self.api = DlsiteAPI(locale=locale) 197 self._throttle = _Throttle(interval) 198 self._semaphore = asyncio.Semaphore(max(concurrency, 1)) 199 self._retries = max(retries, 1) 200 self._timeout = ClientTimeout(total=timeout) 201 self.request_count = 0 202 203 async def __aenter__(self) -> Self: 204 """Enter the client session.""" 205 return self 206 207 async def __aexit__( 208 self, 209 exc_type: type[BaseException] | None, 210 exc: BaseException | None, 211 tb: TracebackType | None, 212 ) -> None: 213 """Close the client session.""" 214 await self.close() 215 216 async def close(self) -> None: 217 """Close the underlying HTTP session.""" 218 await self.api.close() 219 220 async def _get_text(self, url: str, *, params: Mapping[str, str] | None = None) -> str | None: 221 """GET ``url`` and return its body, or ``None`` on a 404/403. 222 223 Retries transient failures with exponential backoff. 224 """ 225 last_error: Exception | None = None 226 for attempt in range(self._retries): 227 async with self._semaphore: 228 await self._throttle.wait() 229 self.request_count += 1 230 try: 231 async with self.api.get( 232 url, 233 params=dict(params or {}), 234 headers={"User-Agent": USER_AGENT}, 235 timeout=self._timeout, 236 ) as response: 237 return await response.text() 238 except ClientResponseError as e: 239 if e.status not in _RETRY_STATUSES: 240 if e.status in {403, 404}: 241 return None 242 raise 243 last_error = e 244 except (ClientError, TimeoutError) as e: 245 last_error = e 246 backoff = 2.0**attempt 247 LOGGER.warning("request failed (%s), retrying in %.0fs: %s", last_error, backoff, url) 248 await asyncio.sleep(backoff) 249 msg = f"giving up on {url}: {last_error}" 250 raise WorkNotFoundError(msg) from last_error 251 252 async def fetch_listing_ids(self, query: SearchQuery, page: int) -> list[str]: 253 """Return the product IDs on ``page`` of the ``query`` listing.""" 254 html = await self._get_text(query.page_url(page)) 255 if html is None: 256 return [] 257 return extract_product_ids(html) 258 259 async def fetch_product_info(self, product_ids: Sequence[str]) -> dict[str, dict[str, Any]]: 260 """Return raw ajax info for up to ~100 product IDs in a single request. 261 262 IDs DLsite does not know about are simply absent from the result. 263 """ 264 if not product_ids: 265 return {} 266 text = await self._get_text(INFO_URL, params={"product_id": ",".join(product_ids)}) 267 if not text: 268 return {} 269 try: 270 data = json.loads(text) 271 except ValueError: 272 LOGGER.warning("malformed info response for %d ids", len(product_ids)) 273 return {} 274 if not isinstance(data, dict): 275 return {} 276 return {k: v for k, v in data.items() if isinstance(v, dict)} 277 278 async def fetch_detail(self, work: Work) -> Work: 279 """Fill in the work-page-only fields of ``work``. 280 281 Raises: 282 WorkNotFoundError: Neither the work nor the announce page exists. 283 """ 284 if parse_work_html is None: # pragma: no cover - dependency fallback 285 return await self.api.get_work(work.product_id) 286 site_id = work.site_id or "maniax" 287 for kind in ("work", "announce"): 288 url = f"https://www.dlsite.com/{site_id}/{kind}/=/product_id/{work.product_id}.html/" 289 html = await self._get_text(url) 290 if html: 291 return replace(work, **parse_detail_html(html)) 292 msg = f"no work or announce page for {work.product_id}" 293 raise WorkNotFoundError(msg)
An HTTP session against DLsite with throttling, retries and counters.
Arguments:
- locale: Optional DLsite locale (e.g.
en_US). Defaults to Japanese. - interval: Minimum seconds between requests.
- concurrency: Maximum number of in-flight requests.
- retries: Attempts per request on transient failures.
- timeout: Per-request timeout in seconds.
187 def __init__( 188 self, 189 *, 190 locale: str | None = None, 191 interval: float = 0.5, 192 concurrency: int = 4, 193 retries: int = 3, 194 timeout: float = 30.0, 195 ) -> None: 196 self.api = DlsiteAPI(locale=locale) 197 self._throttle = _Throttle(interval) 198 self._semaphore = asyncio.Semaphore(max(concurrency, 1)) 199 self._retries = max(retries, 1) 200 self._timeout = ClientTimeout(total=timeout) 201 self.request_count = 0
216 async def close(self) -> None: 217 """Close the underlying HTTP session.""" 218 await self.api.close()
Close the underlying HTTP session.
252 async def fetch_listing_ids(self, query: SearchQuery, page: int) -> list[str]: 253 """Return the product IDs on ``page`` of the ``query`` listing.""" 254 html = await self._get_text(query.page_url(page)) 255 if html is None: 256 return [] 257 return extract_product_ids(html)
Return the product IDs on page of the query listing.
259 async def fetch_product_info(self, product_ids: Sequence[str]) -> dict[str, dict[str, Any]]: 260 """Return raw ajax info for up to ~100 product IDs in a single request. 261 262 IDs DLsite does not know about are simply absent from the result. 263 """ 264 if not product_ids: 265 return {} 266 text = await self._get_text(INFO_URL, params={"product_id": ",".join(product_ids)}) 267 if not text: 268 return {} 269 try: 270 data = json.loads(text) 271 except ValueError: 272 LOGGER.warning("malformed info response for %d ids", len(product_ids)) 273 return {} 274 if not isinstance(data, dict): 275 return {} 276 return {k: v for k, v in data.items() if isinstance(v, dict)}
Return raw ajax info for up to ~100 product IDs in a single request.
IDs DLsite does not know about are simply absent from the result.
278 async def fetch_detail(self, work: Work) -> Work: 279 """Fill in the work-page-only fields of ``work``. 280 281 Raises: 282 WorkNotFoundError: Neither the work nor the announce page exists. 283 """ 284 if parse_work_html is None: # pragma: no cover - dependency fallback 285 return await self.api.get_work(work.product_id) 286 site_id = work.site_id or "maniax" 287 for kind in ("work", "announce"): 288 url = f"https://www.dlsite.com/{site_id}/{kind}/=/product_id/{work.product_id}.html/" 289 html = await self._get_text(url) 290 if html: 291 return replace(work, **parse_detail_html(html)) 292 msg = f"no work or announce page for {work.product_id}" 293 raise WorkNotFoundError(msg)
Fill in the work-page-only fields of work.
Raises:
- WorkNotFoundError: Neither the work nor the announce page exists.
296def batched(items: Sequence[str], size: int) -> Iterable[Sequence[str]]: 297 """Split ``items`` into consecutive chunks of at most ``size``.""" 298 step = max(size, 1) 299 for start in range(0, len(items), step): 300 yield items[start : start + step]
Split items into consecutive chunks of at most size.