Skip to content

Commit b63c3ec

Browse files
SubJunkSubJunk
authored
Improve matching for queries with different languages and years (#1622)
* Improve matching for queries with different languages and years * Made logging take up less space
1 parent 538a47a commit b63c3ec

3 files changed

Lines changed: 76 additions & 78 deletions

File tree

‎src/controllers/media.ts‎

Lines changed: 36 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import MediaMetadata, { MediaMetadataInterface } from '../models/MediaMetadata';
88
import { SeasonMetadataInterface } from '../models/SeasonMetadata';
99
import { SeriesMetadataInterface } from '../models/SeriesMetadata';
1010
import * as externalAPIHelper from '../services/external-api-helper';
11+
import { traceLog } from '../helpers/logging';
1112

1213
const THIRTY_DAYS_IN_MILLISECONDS = 24 * 60 * 60 * 30 * 1000;
1314

@@ -258,15 +259,40 @@ export const getVideoV2 = async(ctx): Promise<MediaMetadataInterface> => {
258259
if (title) {
259260
searchMatch = language ? language + '@' + title : title;
260261
const titleQuery: GetVideoFilter = { searchMatches: { $in: [searchMatch] } };
261-
const titleFailedQuery: FailedLookupsInterface = { title };
262+
263+
// there will be a way to make this automatic but I cbf rn
264+
const titleFailedQuery: {
265+
episode?: string;
266+
failedValidation?: boolean;
267+
imdbID?: string;
268+
language?: string | { $exists: boolean };
269+
season?: string;
270+
startYear?: string;
271+
title?: string;
272+
tmdbID?: number;
273+
type?: string;
274+
year?: string | { $exists: boolean };
275+
count?: number;
276+
reason?: string;
277+
278+
// Added automatically:
279+
createdAt?: string;
280+
updatedAt?: string;
281+
} = { title };
262282

263283
if (language) {
264284
titleFailedQuery.language = language;
285+
} else {
286+
titleFailedQuery.language = { $exists: false };
265287
}
288+
266289
if (year) {
267290
titleQuery.year = year;
268291
titleFailedQuery.year = year;
292+
} else {
293+
titleFailedQuery.year = { $exists: false };
269294
}
295+
270296
if (episode) {
271297
titleQuery.episode = episode;
272298
titleFailedQuery.episode = episode;
@@ -281,39 +307,32 @@ export const getVideoV2 = async(ctx): Promise<MediaMetadataInterface> => {
281307

282308
const existingResult = await MediaMetadata.findOne({ $or: query }, null, { lean: true }).exec();
283309
if (existingResult) {
284-
if (process.env.VERBOSE === 'true') {
285-
console.trace('found existingResult', query, existingResult);
286-
}
310+
traceLog('found existingResult', { query, existingResult });
311+
287312
// we have an existing metadata record, so return it
288313
return ctx.body = existingResult;
289314
}
290315

291316
const existingFailedResult = await FailedLookups.findOne({ $or: failedQuery }, null, { lean: true }).exec();
292317
if (existingFailedResult) {
293318
// we have an existing failure record, so increment it, and throw not found error
294-
if (process.env.VERBOSE === 'true') {
295-
console.trace('found existingFailedResult', existingFailedResult, failedQuery);
296-
}
319+
traceLog('found existingFailedResult', { existingFailedResult, failedQuery });
297320
await FailedLookups.updateOne({ _id: existingFailedResult._id }, { $inc: { count: 1 } }).exec();
298321
throw new MediaNotFoundError();
299322
}
300323

301324
// the database does not have a record of this file, so begin search for metadata on TMDB.
302325

303-
const failedLookupQuery: FailedLookupsInterface = { episode, imdbID, season, title, year };
326+
const failedLookupQuery: FailedLookupsInterface = { episode, imdbID, season, title, year, language };
304327

305328
let tmdbData: MediaMetadataInterface;
306329
try {
307330
tmdbData = await externalAPIHelper.getFromTMDBAPI(title, language, imdbIdToSearch, yearNumber, seasonNumber, episodeNumbers);
308331
imdbIdToSearch = imdbIdToSearch || tmdbData?.imdbID;
309-
if (process.env.VERBOSE === 'true') {
310-
console.trace('found tmdbData and imdbIdToSearch', query, tmdbData, imdbIdToSearch);
311-
}
332+
traceLog('found tmdbData and imdbIdToSearch', { query, tmdbData, imdbIdToSearch });
312333
} catch (err) {
313334
if (err instanceof RateLimitError) {
314-
if (process.env.VERBOSE === 'true') {
315-
console.trace(err);
316-
}
335+
traceLog(err);
317336
throw err;
318337
}
319338

@@ -329,17 +348,13 @@ export const getVideoV2 = async(ctx): Promise<MediaMetadataInterface> => {
329348
if (!imdbID && imdbIdToSearch) {
330349
const existingResult = await MediaMetadata.findOne({ imdbID: imdbIdToSearch }, null, { lean: true }).exec();
331350
if (existingResult) {
332-
if (process.env.VERBOSE === 'true') {
333-
console.trace('found existingResult from IMDb ID from TMDB', existingResult, imdbIdToSearch);
334-
}
351+
traceLog('found existingResult from IMDb ID from TMDB', { existingResult, imdbIdToSearch });
335352
return ctx.body = await addSearchMatchByIMDbID(imdbIdToSearch, searchMatch);
336353
}
337354
}
338355

339356
if (!tmdbData || _.isEmpty(tmdbData)) {
340-
if (process.env.VERBOSE === 'true') {
341-
console.trace('No data was found on TMDB for this query', title, language, imdbIdToSearch, yearNumber, seasonNumber, episodeNumbers);
342-
}
357+
traceLog('No data was found on TMDB for this query', { title, language, imdbIdToSearch, yearNumber, seasonNumber, episodeNumbers });
343358
const reason = `getVideoV2 got no tmdb data for ${title}, ${language}, ${imdbIdToSearch}, ${yearNumber}, ${seasonNumber}, ${episodeNumbers}`;
344359
await FailedLookups.updateOne(failedLookupQuery, { $inc: { count: 1 }, reason }, { upsert: true, setDefaultsOnInsert: true }).exec();
345360
throw new MediaNotFoundError();
@@ -395,9 +410,7 @@ export const getVideoV2 = async(ctx): Promise<MediaMetadataInterface> => {
395410
return ctx.body = leanMeta;
396411
} catch (e) {
397412
console.error(e, tmdbData);
398-
if (process.env.VERBOSE === 'true') {
399-
console.trace('No data was found on TMDB for this query in final getVideoV2 catch', e);
400-
}
413+
traceLog('No data was found on TMDB for this query in final getVideoV2 catch', e);
401414
const reason = `getVideoV2 caught an exception ${e}, for ${tmdbData}`;
402415
await FailedLookups.updateOne(failedLookupQuery, { $inc: { count: 1 }, reason }, { upsert: true, setDefaultsOnInsert: true }).exec();
403416
throw new MediaNotFoundError();

‎src/helpers/logging.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
/**
2+
* Log text in trace mode if VERBOSE env var is true
3+
*/
4+
export const traceLog = (text: string | Error, extraInfo: unknown = null) => {
5+
if (process.env.VERBOSE === 'true') {
6+
console.trace(text, extraInfo);
7+
}
8+
}

‎src/services/external-api-helper.ts‎

Lines changed: 32 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { Episode, EpisodeRequest, ExternalId, SearchMovieRequest, SearchTvReques
66

77
import { tmdb } from './tmdb-api';
88
import { ValidationError } from '../helpers/customErrors';
9+
import { traceLog } from '../helpers/logging';
910
import CollectionMetadata, { CollectionMetadataInterface } from '../models/CollectionMetadata';
1011
import FailedLookups, { FailedLookupsInterface } from '../models/FailedLookups';
1112
import LocalizeMetadata, { LocalizeMetadataInterface } from '../models/LocalizeMetadata';
@@ -106,7 +107,7 @@ export const getSeriesMetadata = async(
106107
title?: string;
107108
tmdbID?: number;
108109
type?: string;
109-
year?: string;
110+
year?: string | { $exists: boolean };
110111
count?: number;
111112
reason?: string;
112113

@@ -175,9 +176,7 @@ export const getSeriesMetadata = async(
175176

176177
// Return early for previously-failed lookups
177178
if (await FailedLookups.findOne(failedLookupQuery, '_id', { lean: true }).exec()) {
178-
if (process.env.VERBOSE === 'true') {
179-
console.trace('Found previously-failed lookup', failedLookupQuery);
180-
}
179+
traceLog('Found previously-failed lookup', failedLookupQuery);
181180
await FailedLookups.updateOne(failedLookupQuery, { $inc: { count: 1 } }).exec();
182181

183182
// Also store a failed result for the title that the client sent
@@ -189,9 +188,7 @@ export const getSeriesMetadata = async(
189188
}
190189

191190
// Return any previous match
192-
if (process.env.VERBOSE === 'true') {
193-
console.trace('Looking for TV series in db', parsedTitle);
194-
}
191+
traceLog('Looking for TV series in db', { parsedTitle });
195192
const seriesMetadata = await SeriesMetadata.findOne(titleQuery, null, { lean: true }).sort(sortBy).exec();
196193
if (seriesMetadata) {
197194
// Also cache the result for the title that the client sent, if this is an automatic re-attempt with an appended year (see below)
@@ -202,37 +199,31 @@ export const getSeriesMetadata = async(
202199
{ returnDocument: 'after', lean: true },
203200
).exec();
204201
}
205-
if (process.env.VERBOSE === 'true') {
206-
console.trace('Found TV series', seriesMetadata);
207-
}
202+
203+
traceLog('Found TV series', seriesMetadata);
208204

209205
return seriesMetadata;
210206
}
211207

212208
// Start TMDB lookups
213-
if (process.env.VERBOSE === 'true') {
214-
console.trace('Looking for TV seriesTMDBID on TMDB', parsedTitle);
215-
}
209+
traceLog('Looking for TV seriesTMDBID on TMDB', { parsedTitle });
210+
216211
const seriesTMDBID = await getSeriesTMDBIDFromTMDBAPI(null, parsedTitle, language, yearNumber);
217212
if (seriesTMDBID) {
218-
if (process.env.VERBOSE === 'true') {
219-
console.trace('Found TV seriesTMDBID for parsedTitle', seriesTMDBID);
220-
}
213+
traceLog('Found TV seriesTMDBID for parsedTitle', { seriesTMDBID });
214+
221215
// See if we have an existing record for the now-known media.
222216
const existingResult = await SeriesMetadata.findOne({ tmdbID: seriesTMDBID }, null, { lean: true }).exec();
223217
if (existingResult) {
224-
if (process.env.VERBOSE === 'true') {
225-
console.trace('Found existingResult for parsedTitle', existingResult);
226-
}
218+
traceLog('Found existingResult for parsedTitle', existingResult);
219+
227220
return await SeriesMetadata.findOneAndUpdate(
228221
{ tmdbID: seriesTMDBID },
229222
{ $addToSet: { searchMatches: searchMatch } },
230223
{ returnDocument: 'after', lean: true },
231224
).exec();
232225
} else {
233-
if (process.env.VERBOSE === 'true') {
234-
console.trace('No existingResult for parsedTitle', existingResult);
235-
}
226+
traceLog('No existingResult for parsedTitle', existingResult);
236227
}
237228

238229
// We do not have an existing record for that series, get the full result from the TMDB API
@@ -241,13 +232,12 @@ export const getSeriesMetadata = async(
241232
id: seriesTMDBID,
242233
};
243234

244-
if (process.env.VERBOSE === 'true') {
245-
console.trace('Looking for series on TMDB', parsedTitle);
246-
}
235+
traceLog('Looking for series on TMDB', { parsedTitle });
236+
247237
const tmdbResponse = await tmdb.tvInfo(seriesRequest);
248-
if (process.env.VERBOSE === 'true') {
249-
console.trace('Found series on TMDB', tmdbResponse);
250-
}
238+
239+
traceLog('Found series on TMDB', tmdbResponse);
240+
251241
tmdbData = mapper.parseTMDBAPISeriesResponse(tmdbResponse);
252242
}
253243
// End TMDB lookups
@@ -402,44 +392,35 @@ export const getFromTMDBAPI = async(movieOrSeriesTitle?: string, language?: stri
402392
const episodeIMDbID = movieOrEpisodeIMDbID;
403393
let seriesTMDBID: string | number;
404394
if (episodeIMDbID) {
405-
if (process.env.VERBOSE === 'true') {
406-
console.trace('Looking for an episode with the IMDb ID', episodeIMDbID);
407-
}
395+
traceLog('Looking for an episode with the IMDb ID', { episodeIMDbID });
396+
408397
const findResult = await tmdb.find({ id: episodeIMDbID, external_source: ExternalId.ImdbId });
409398
if (findResult?.tv_episode_results && findResult?.tv_episode_results[0]) {
410399
const tvEpisodeResult = findResult.tv_episode_results[0] as SimpleEpisode;
411400
seriesTMDBID = tvEpisodeResult?.show_id;
412-
if (process.env.VERBOSE === 'true') {
413-
console.trace('Found tvEpisodeResult and seriesTMDBID', tvEpisodeResult, seriesTMDBID);
414-
}
401+
402+
traceLog('Found tvEpisodeResult and seriesTMDBID', { tvEpisodeResult, seriesTMDBID });
415403
} else {
416-
if (process.env.VERBOSE === 'true') {
417-
console.trace('Did not find an episode with the IMDb ID', episodeIMDbID);
418-
}
404+
traceLog('Did not find an episode with the IMDb ID', { episodeIMDbID });
419405
}
420406
} else {
421-
if (process.env.VERBOSE === 'true') {
422-
console.trace('Looking for seriesTMDBID with', movieOrSeriesTitle, language, yearString);
423-
}
407+
traceLog('Looking for seriesTMDBID with', { movieOrSeriesTitle, language, yearString });
408+
424409
const seriesMetadata = await getSeriesMetadata(null, movieOrSeriesTitle, language, yearString);
425410
seriesTMDBID = seriesMetadata?.tmdbID;
426411
}
427412

428413
if (!seriesTMDBID) {
429-
if (process.env.VERBOSE === 'true') {
430-
console.trace('Did not find seriesTMDBID with', movieOrSeriesTitle, language, yearString);
431-
}
414+
traceLog('Did not find seriesTMDBID with', { movieOrSeriesTitle, language, yearString });
415+
432416
return null;
433417
} else {
434-
if (process.env.VERBOSE === 'true') {
435-
console.trace('Found seriesTMDBID ' + seriesTMDBID + 'with', movieOrSeriesTitle, language, yearString);
436-
}
418+
traceLog('Found seriesTMDBID ' + seriesTMDBID + 'with', { movieOrSeriesTitle, language, yearString });
437419
}
438420

439421
for (let i = 0; i < episodeNumbers.length; i++) {
440-
if (process.env.VERBOSE === 'true') {
441-
console.trace('Looking for episode number ' + episodeNumbers[i] + 'with', seriesTMDBID, seasonNumber);
442-
}
422+
traceLog('Looking for episode number ' + episodeNumbers[i] + 'with', { seriesTMDBID, seasonNumber });
423+
443424
const episodeRequest: EpisodeRequest = {
444425
append_to_response: 'images,external_ids,credits',
445426
episode_number: episodeNumbers[i],
@@ -461,16 +442,12 @@ export const getFromTMDBAPI = async(movieOrSeriesTitle?: string, language?: stri
461442
if (tmdbSeriesData?.imdb_id) {
462443
metadata.seriesIMDbID = tmdbSeriesData.imdb_id;
463444
}
464-
if (process.env.VERBOSE === 'true') {
465-
console.trace('Found metadata', metadata);
466-
}
445+
traceLog('Found metadata', metadata);
467446
} else {
468447
metadata.title = metadata.title ? metadata.title + ' & ' + tmdbData.name : tmdbData.name;
469448
}
470449
} else {
471-
if (process.env.VERBOSE === 'true') {
472-
console.trace('Did not find tmdbData from', episodeRequest);
473-
}
450+
traceLog('Did not find tmdbData from', episodeRequest);
474451
}
475452
}
476453
} else {

0 commit comments

Comments
 (0)