eben0 / snooshift Goto Github PK

const searchParams = {
	size: 25,
	sort_type: 'new',
	subreddit: 'askscience'
};
const res = await snoo.searchSubmissions(searchParams); // zero results

@Injectable()
export class WebScraperService {
  private readonly logger = new Logger(WebScraperService.name);
  private readonly subreddits = ["dankmemes", "memes"];
  private readonly exts = [".jpg", ".jpeg", ".png"];
  private readonly snoo = new SnooShift();
  private redditScraperMutex: boolean = false;
  private imgflipScraperMutex: boolean = false;

  constructor(
    private readonly redditMemeService: RedditMemeService,
    private readonly redditorService: RedditorService,
    private readonly imgflipTemplateService: ImgflipTemplateService
  ) {}

  // @Cron(CRON_SCHEDULES[ECronJobRegistry.RedditMemeScrapper], { name: ECronJobRegistry.RedditMemeScrapper })
  async redditorMemeScrapper() {
    if (this.redditScraperMutex) return;
    else this.redditScraperMutex = true;
    for (const subreddit of this.subreddits) {
      this.logger.log(`RUNNING REDDIT MEME SCRAPPER: r/${subreddit}`);
      try {
        await this.scrapeSubReddit({ subreddit });
      } catch (error) {
        this.logger.error(error.message, error.stack);
      }
    }
    this.logger.log("DONE REDDIT MEME SCRAPPER");
    this.redditScraperMutex = false;
  }

  private async scrapeSubReddit({ subreddit, gracePeriod = 7 }: { subreddit: string; gracePeriod?: number }) {
    const endAt = dayjs().startOf("h").subtract(gracePeriod, "d");
    const maxCreatedAt = await this.redditMemeService.repo.max("createdAt");
    let startAt = maxCreatedAt && maxCreatedAt.result ? dayjs(maxCreatedAt.result) : dayjs().startOf("d").subtract(62, "day");
    while (startAt < endAt) {
      this.logger.log(`SCRAPPING STARTING AT ${startAt}`);
      const after = startAt.unix(),
        before = endAt.unix();
      const unfilteredSubmissions = (await this.snoo.searchSubmissions({
        subreddit,
        after,
        before,
        size: 100,
        stickied: false,
      })) as Submission[];
      // console.log("unfilteredSubmissions", unfilteredSubmissions);
      // throw new Error("check");
      const submissions = unfilteredSubmissions.filter(({ url }) => this.exts.some((ext) => url.endsWith(ext)));
      const usernames = submissions.map(({ author_fullname }) => author_fullname);
      const redditors = await this.redditorService.repo.find({ where: { username: In(usernames) } });
      const usernameToOldRedditor = redditors.reduce<Record<string, RedditorEntity>>(
        (prev, redditor) => ({ [redditor.username]: redditor, ...prev }),
        {}
      );
      const urls = submissions.map(({ url }) => url);
      const redditMemes = await this.redditMemeService.repo.find({ select: ["url"], where: { url: In(urls) } });
      const urlSet = new Set(redditMemes.map(({ url }) => url));
      const dedupSubmissions = submissions.filter(({ url }) => url && !urlSet.has(url));
      const usernameToNewRedditor = dedupSubmissions
        .filter(({ author_fullname }) => !usernameToOldRedditor[author_fullname])
        .reduce<Record<string, RedditorEntity>>(
          (prev, { author_fullname }) => ({ [author_fullname]: this.redditorService.repo.create({ username: author_fullname }), ...prev }),
          {}
        );
      await this.redditorService.repo.save(Object.values(usernameToNewRedditor));
      const usernameToRedditor = { ...usernameToOldRedditor, ...usernameToNewRedditor };

      const urlToNewRedditMeme = dedupSubmissions.reduce<Record<string, RedditMemeEntity>>(
        (prev, { id, num_comments, title, score, created_utc, upvote_ratio, url, author_fullname }) => ({
          [url]: this.redditMemeService.repo.create({
            redditId: id,
            numComments: num_comments,
            upvotes: score,
            createdAt: dayjs(created_utc * 1000).toDate(),
            downvotes: Math.round(score / upvote_ratio) - score,
            title,
            url,
            upvoteRatio: upvote_ratio,
            redditorId: usernameToRedditor[author_fullname].id,
            subreddit,
          }),
          ...prev,
        }),
        {}
      );
      await this.redditMemeService.repo.save(Object.values(urlToNewRedditMeme));
      startAt = dayjs(1000 * Math.max(...submissions.map(({ created_utc }) => created_utc)));
      await new Promise((r) => setTimeout(r, 5000));
    }
  }
 }