Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""LICENSE
2Copyright 2020 Hermann Krumrey <hermann@krumreyh.com>
4This file is part of otaku-info-web.
6otaku-info-web is free software: you can redistribute it and/or modify
7it under the terms of the GNU General Public License as published by
8the Free Software Foundation, either version 3 of the License, or
9(at your option) any later version.
11otaku-info-web is distributed in the hope that it will be useful,
12but WITHOUT ANY WARRANTY; without even the implied warranty of
13MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14GNU General Public License for more details.
16You should have received a copy of the GNU General Public License
17along with otaku-info-web. If not, see <http://www.gnu.org/licenses/>.
18LICENSE"""
20import time
21from typing import Dict, List, Optional, Tuple
22from sqlalchemy.exc import IntegrityError
23from puffotter.flask.base import db, app
24from otaku_info_web.db.MediaItem import MediaItem
25from otaku_info_web.db.MediaId import MediaId
26from otaku_info_web.utils.enums import ListService, MediaType
27from otaku_info_web.utils.mangadex.api import get_external_ids
28from otaku_info_web.utils.anilist.api import load_media_info
31def load_id_mappings():
32 """
33 Goes through mangadex IDs sequentially and stores ID mappings for
34 these entries if found
35 :return: None
36 """
37 endcounter = 0
39 anilist_ids, existing_ids = load_db_content()
41 if len(existing_ids) > 0:
42 mangadex_id = max(existing_ids)
43 else:
44 mangadex_id = 0
46 while True:
47 mangadex_id += 1
49 if mangadex_id % 100 == 0:
50 app.logger.debug("Refreshing mangadex cache")
51 anilist_ids, existing_ids = load_db_content()
53 app.logger.debug(f"Probing mangadex id {mangadex_id}")
55 other_ids = get_external_ids(mangadex_id)
57 if other_ids is None:
58 endcounter += 1
59 if endcounter > 1000:
60 break
61 else:
62 continue
63 else:
64 endcounter = 0
66 store_ids(existing_ids, anilist_ids, mangadex_id, other_ids)
67 app.logger.info("Reached end of mangadex ID range")
70def load_db_content() -> Tuple[
71 Dict[str, MediaId],
72 Dict[int, List[ListService]]
73]:
74 """
75 Loads the existing data from the database.
76 By doing this as few times as possible, we can greatly improve performance
77 :return: The anilist IDs, The mangadex IDs mapped to other existing IDs
78 """
79 start = time.time()
80 app.logger.debug("Starting caching of db data for mangadex ID mapping")
82 all_ids: List[MediaId] = [
83 x for x in
84 MediaId.query.join(MediaItem).all()
85 if x.media_item.media_type == MediaType.MANGA
86 ]
87 anilist_ids: Dict[str, MediaId] = {
88 x.service_id: x
89 for x in all_ids
90 if x.service == ListService.ANILIST
91 }
93 mangadex_idmap: Dict[int, int] = {}
95 existing_ids: Dict[int, List[ListService]] = {}
96 for media_id in all_ids:
97 media_item_id = media_id.media_item_id
98 if media_item_id not in existing_ids:
99 existing_ids[media_item_id] = []
100 existing_ids[media_item_id].append(media_id)
101 if media_id.service == ListService.MANGADEX:
102 mangadex_idmap[media_item_id] = int(media_id.service_id)
104 mapped_existing_ids = {
105 mangadex_idmap[key]: value
106 for key, value in existing_ids.items()
107 if key in mangadex_idmap
108 }
110 app.logger.info(f"Finished caching of db data for mangadex ID mapping "
111 f"in {time.time() - start}s")
112 return anilist_ids, mapped_existing_ids
115def store_ids(
116 existing_ids: Dict[int, List[ListService]],
117 anilist_ids: Dict[str, MediaId],
118 mangadex_id: int,
119 other_ids: Dict[ListService, str]
120):
121 """
122 Stores the fetched IDs in the database
123 :param existing_ids: A dictionary mapping mangadex IDs to existing
124 list service types
125 :param anilist_ids: Dictionary mapping anilist IDs to media IDs
126 :param mangadex_id: The mangadex ID
127 :param other_ids: The other IDs
128 :return: None
129 """
130 if ListService.ANILIST not in other_ids:
131 return
133 existing_services = existing_ids.get(mangadex_id, [])
134 existing_ids[mangadex_id] = existing_services
135 anilist_id = other_ids[ListService.ANILIST]
137 if anilist_id not in anilist_ids:
138 media_item = create_anilist_media_item(int(anilist_id))
139 if media_item is None:
140 return
141 else:
142 media_item_id = media_item.id
143 else:
144 media_item_id = anilist_ids[anilist_id].media_item_id
145 existing_services.append(ListService.ANILIST)
147 app.logger.debug(f"Storing external IDS for mangadex id {mangadex_id}")
149 for list_service, _id in other_ids.items():
150 if list_service not in existing_services:
151 media_id = MediaId(
152 media_item_id=media_item_id,
153 service=list_service,
154 service_id=_id
155 )
156 db.session.add(media_id)
157 existing_ids[mangadex_id].append(list_service)
158 if list_service == ListService.ANILIST:
159 anilist_ids[_id] = media_id
161 try:
162 db.session.commit()
163 except IntegrityError:
164 # Since mangadex has some entries that point to the exact same anilist
165 # media item, we may sometimes encounter cases where we have two
166 # mangadex IDs for one anilist ID.
167 # By ignoring errors here, only the first mangadex ID will be stored.
168 # An example for this issue is Genshiken (961) and its
169 # sequel Genshiken Nidaime (962)
170 db.session.rollback()
171 app.logger.warning(f"Couldn't add mangadex ID {mangadex_id}")
174def create_anilist_media_item(anilist_id: int) -> Optional[MediaItem]:
175 """
176 Creates an anilist media item using an anilist ID, fetching the data using
177 the anilist API
178 :param anilist_id: The anilist ID of the media
179 :return: The generated Media Item
180 """
181 anilist_entry = load_media_info(anilist_id, MediaType.MANGA)
182 if anilist_entry is None:
183 return None
184 media_item = MediaItem(
185 media_type=MediaType.MANGA,
186 media_subtype=anilist_entry.media_subtype,
187 english_title=anilist_entry.english_title,
188 romaji_title=anilist_entry.romaji_title,
189 cover_url=anilist_entry.cover_url,
190 latest_release=anilist_entry.latest_release,
191 releasing_state=anilist_entry.releasing_state
192 )
193 db.session.add(media_item)
195 try:
196 db.session.commit()
197 except IntegrityError:
198 db.session.rollback()
199 app.logger.warning(f"Failed to add anilist manga entry "
200 f"(ID={anilist_id})")
201 return None
203 return media_item