Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1"""LICENSE 

2Copyright 2020 Hermann Krumrey <hermann@krumreyh.com> 

3 

4This file is part of otaku-info-web. 

5 

6otaku-info-web is free software: you can redistribute it and/or modify 

7it under the terms of the GNU General Public License as published by 

8the Free Software Foundation, either version 3 of the License, or 

9(at your option) any later version. 

10 

11otaku-info-web is distributed in the hope that it will be useful, 

12but WITHOUT ANY WARRANTY; without even the implied warranty of 

13MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

14GNU General Public License for more details. 

15 

16You should have received a copy of the GNU General Public License 

17along with otaku-info-web. If not, see <http://www.gnu.org/licenses/>. 

18LICENSE""" 

19 

20import time 

21from typing import Dict, List, Optional, Tuple 

22from sqlalchemy.exc import IntegrityError 

23from puffotter.flask.base import db, app 

24from otaku_info_web.db.MediaItem import MediaItem 

25from otaku_info_web.db.MediaId import MediaId 

26from otaku_info_web.utils.enums import ListService, MediaType 

27from otaku_info_web.utils.mangadex.api import get_external_ids 

28from otaku_info_web.utils.anilist.api import load_media_info 

29 

30 

31def load_id_mappings(): 

32 """ 

33 Goes through mangadex IDs sequentially and stores ID mappings for 

34 these entries if found 

35 :return: None 

36 """ 

37 endcounter = 0 

38 

39 anilist_ids, existing_ids = load_db_content() 

40 

41 if len(existing_ids) > 0: 

42 mangadex_id = max(existing_ids) 

43 else: 

44 mangadex_id = 0 

45 

46 while True: 

47 mangadex_id += 1 

48 

49 if mangadex_id % 100 == 0: 

50 app.logger.debug("Refreshing mangadex cache") 

51 anilist_ids, existing_ids = load_db_content() 

52 

53 app.logger.debug(f"Probing mangadex id {mangadex_id}") 

54 

55 other_ids = get_external_ids(mangadex_id) 

56 

57 if other_ids is None: 

58 endcounter += 1 

59 if endcounter > 1000: 

60 break 

61 else: 

62 continue 

63 else: 

64 endcounter = 0 

65 

66 store_ids(existing_ids, anilist_ids, mangadex_id, other_ids) 

67 app.logger.info("Reached end of mangadex ID range") 

68 

69 

70def load_db_content() -> Tuple[ 

71 Dict[str, MediaId], 

72 Dict[int, List[ListService]] 

73]: 

74 """ 

75 Loads the existing data from the database. 

76 By doing this as few times as possible, we can greatly improve performance 

77 :return: The anilist IDs, The mangadex IDs mapped to other existing IDs 

78 """ 

79 start = time.time() 

80 app.logger.debug("Starting caching of db data for mangadex ID mapping") 

81 

82 all_ids: List[MediaId] = [ 

83 x for x in 

84 MediaId.query.join(MediaItem).all() 

85 if x.media_item.media_type == MediaType.MANGA 

86 ] 

87 anilist_ids: Dict[str, MediaId] = { 

88 x.service_id: x 

89 for x in all_ids 

90 if x.service == ListService.ANILIST 

91 } 

92 

93 mangadex_idmap: Dict[int, int] = {} 

94 

95 existing_ids: Dict[int, List[ListService]] = {} 

96 for media_id in all_ids: 

97 media_item_id = media_id.media_item_id 

98 if media_item_id not in existing_ids: 

99 existing_ids[media_item_id] = [] 

100 existing_ids[media_item_id].append(media_id) 

101 if media_id.service == ListService.MANGADEX: 

102 mangadex_idmap[media_item_id] = int(media_id.service_id) 

103 

104 mapped_existing_ids = { 

105 mangadex_idmap[key]: value 

106 for key, value in existing_ids.items() 

107 if key in mangadex_idmap 

108 } 

109 

110 app.logger.info(f"Finished caching of db data for mangadex ID mapping " 

111 f"in {time.time() - start}s") 

112 return anilist_ids, mapped_existing_ids 

113 

114 

115def store_ids( 

116 existing_ids: Dict[int, List[ListService]], 

117 anilist_ids: Dict[str, MediaId], 

118 mangadex_id: int, 

119 other_ids: Dict[ListService, str] 

120): 

121 """ 

122 Stores the fetched IDs in the database 

123 :param existing_ids: A dictionary mapping mangadex IDs to existing 

124 list service types 

125 :param anilist_ids: Dictionary mapping anilist IDs to media IDs 

126 :param mangadex_id: The mangadex ID 

127 :param other_ids: The other IDs 

128 :return: None 

129 """ 

130 if ListService.ANILIST not in other_ids: 

131 return 

132 

133 existing_services = existing_ids.get(mangadex_id, []) 

134 existing_ids[mangadex_id] = existing_services 

135 anilist_id = other_ids[ListService.ANILIST] 

136 

137 if anilist_id not in anilist_ids: 

138 media_item = create_anilist_media_item(int(anilist_id)) 

139 if media_item is None: 

140 return 

141 else: 

142 media_item_id = media_item.id 

143 else: 

144 media_item_id = anilist_ids[anilist_id].media_item_id 

145 existing_services.append(ListService.ANILIST) 

146 

147 app.logger.debug(f"Storing external IDS for mangadex id {mangadex_id}") 

148 

149 for list_service, _id in other_ids.items(): 

150 if list_service not in existing_services: 

151 media_id = MediaId( 

152 media_item_id=media_item_id, 

153 service=list_service, 

154 service_id=_id 

155 ) 

156 db.session.add(media_id) 

157 existing_ids[mangadex_id].append(list_service) 

158 if list_service == ListService.ANILIST: 

159 anilist_ids[_id] = media_id 

160 

161 try: 

162 db.session.commit() 

163 except IntegrityError: 

164 # Since mangadex has some entries that point to the exact same anilist 

165 # media item, we may sometimes encounter cases where we have two 

166 # mangadex IDs for one anilist ID. 

167 # By ignoring errors here, only the first mangadex ID will be stored. 

168 # An example for this issue is Genshiken (961) and its 

169 # sequel Genshiken Nidaime (962) 

170 db.session.rollback() 

171 app.logger.warning(f"Couldn't add mangadex ID {mangadex_id}") 

172 

173 

174def create_anilist_media_item(anilist_id: int) -> Optional[MediaItem]: 

175 """ 

176 Creates an anilist media item using an anilist ID, fetching the data using 

177 the anilist API 

178 :param anilist_id: The anilist ID of the media 

179 :return: The generated Media Item 

180 """ 

181 anilist_entry = load_media_info(anilist_id, MediaType.MANGA) 

182 if anilist_entry is None: 

183 return None 

184 media_item = MediaItem( 

185 media_type=MediaType.MANGA, 

186 media_subtype=anilist_entry.media_subtype, 

187 english_title=anilist_entry.english_title, 

188 romaji_title=anilist_entry.romaji_title, 

189 cover_url=anilist_entry.cover_url, 

190 latest_release=anilist_entry.latest_release, 

191 releasing_state=anilist_entry.releasing_state 

192 ) 

193 db.session.add(media_item) 

194 

195 try: 

196 db.session.commit() 

197 except IntegrityError: 

198 db.session.rollback() 

199 app.logger.warning(f"Failed to add anilist manga entry " 

200 f"(ID={anilist_id})") 

201 return None 

202 

203 return media_item