♻️ refactor: Feed Crawler, Server 환경 변수 이름 및 위치 통일, 로깅 범위 결정 Docker Compose (5)

client/.gitignore

@@ -11,6 +11,7 @@ node_modules
 dist
 dist-ssr
 *.local
+!Dockerfile.local
 
 !.env.local
 !Dockerfile.local

feed-crawler/.gitignore

@@ -1,5 +1,7 @@
 /node_modules
-.env
+.env.*
+!.env.local
 /dist
-feed-crawler.log
-/test/coverage
+/logs
+/test/coverage
+.env

feed-crawler/env/.env.local

@@ -0,0 +1,10 @@
+DB_HOST=mysql-db
+DB_NAME=denamu
+DB_USER=denamu-db-user
+DB_PASS=denamu-db-pw
+DB_TABLE=rss_accept
+TIME_INTERVAL=30
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_USERNAME=denamu-redis-user
+REDIS_PASSWORD=denamu-redis-pw

feed-crawler/src/common/constant.ts

@@ -1,19 +1,14 @@
-import * as dotenv from 'dotenv';
-
-dotenv.config({
-  path: process.env.NODE_ENV === 'production' ? 'feed-crawler/.env' : '.env',
-});
 export const CONNECTION_LIMIT = 50;
 export const redisConstant = {
   FEED_RECENT_ALL_KEY: 'feed:recent:*',
   FEED_AI_QUEUE: `feed:ai:queue`,
 };
 
 export const ONE_MINUTE = 60 * 1000;
-export const INTERVAL =
-  process.env.NODE_ENV === 'test'
-    ? parseInt(process.env.TEST_TIME_INTERVAL)
-    : parseInt(process.env.TIME_INTERVAL);
+export const TIME_INTERVAL =
+  process.env.NODE_ENV !== 'test'
+    ? parseInt(process.env.TIME_INTERVAL)
+    : Number.MAX_SAFE_INTEGER;
 
 export const FEED_AI_SUMMARY_IN_PROGRESS_MESSAGE = `아직 AI가 요약을 진행중인 게시글 이에요! 💭`;
 

feed-crawler/src/common/env-load.ts

@@ -0,0 +1,10 @@
+import * as dotenv from "dotenv";
+
+dotenv.config({
+  path:
+    {
+      PROD: `${process.cwd()}/env/.env.prod`,
+      LOCAL: `${process.cwd()}/env/.env.local`,
+      DEV: `${process.cwd()}/env/.env.local`,
+    }[process.env.NODE_ENV] || "",
+});

feed-crawler/src/common/logger.ts

@@ -7,19 +7,16 @@ const logFormat = printf(({ level, message, timestamp }) => {
 });
 
 const transports = [];
-transports.push(new winston.transports.Console());
-if (process.env.NODE_ENV !== 'test') {
+if (process.env.NODE_ENV === 'LOCAL' || process.env.NODE_ENV === 'PROD') {
   transports.push(
-    new winston.transports.File({
-      filename: `${
-        process.env.NODE_ENV === 'production'
-          ? 'feed-crawler/logs/feed-crawler.log'
-          : 'logs/feed-crawler.log'
-      }`,
-    }),
+    new winston.transports.File({ filename: 'logs/feed-crawler.log' }),
   );
 }
 
+if (process.env.NODE_ENV === 'LOCAL' || process.env.NODE_ENV === 'DEV') {
+  transports.push(new winston.transports.Console());
+}
+
 const logger = winston.createLogger({
   level: 'info',
   format: combine(
@@ -28,6 +25,7 @@ const logger = winston.createLogger({
     logFormat,
   ),
   transports: transports,
+  silent: process.env.NODE_ENV === 'test',
 });
 
 export default logger;

feed-crawler/src/common/mysql-access.ts

@@ -1,14 +1,9 @@
-import * as dotenv from 'dotenv';
 import * as mysql from 'mysql2/promise';
 import { CONNECTION_LIMIT } from './constant';
 import { PoolConnection } from 'mysql2/promise';
 import { DatabaseConnection } from '../types/database-connection';
 import logger from './logger';
 
-dotenv.config({
-  path: process.env.NODE_ENV === 'production' ? 'feed-crawler/.env' : '.env',
-});
-
 export class MySQLConnection implements DatabaseConnection {
   private pool: mysql.Pool;
   private nameTag: string;
@@ -38,7 +33,7 @@ export class MySQLConnection implements DatabaseConnection {
       logger.error(
         `${this.nameTag} 쿼리 ${query} 실행 중 오류 발생
           오류 메시지: ${error.message}
-          스택 트레이스: ${error.stack}`,
+          스택 트레이스: ${error.stack}`
       );
     } finally {
       if (connection) {
@@ -48,7 +43,7 @@ export class MySQLConnection implements DatabaseConnection {
           logger.error(
             `${this.nameTag} connection release 중 오류 발생
             오류 메시지: ${error.message}
-            스택 트레이스: ${error.stack}`,
+            스택 트레이스: ${error.stack}`
           );
         }
       }

feed-crawler/src/common/redis-access.ts

@@ -1,13 +1,8 @@
-import * as dotenv from 'dotenv';
 import Redis, { ChainableCommander } from 'ioredis';
 import Redis_Mock from 'ioredis-mock';
 import logger from '../common/logger';
 import { injectable } from 'tsyringe';
 
-dotenv.config({
-  path: process.env.NODE_ENV === 'production' ? 'feed-crawler/.env' : '.env',
-});
-
 @injectable()
 export class RedisConnection {
   private redis: Redis;

feed-crawler/src/common/rss-parser.ts

@@ -0,0 +1,56 @@
+import logger from "./logger";
+import { parse } from "node-html-parser";
+import { unescape } from "html-escaper";
+
+export class RssParser {
+  async getThumbnailUrl(feedUrl: string) {
+    const response = await fetch(feedUrl, {
+      headers: {
+        Accept: "text/html",
+      },
+    });
+    if (!response.ok) {
+      throw new Error(`${feedUrl}에 GET 요청 실패`);
+    }
+
+    const htmlData = await response.text();
+    const htmlRootElement = parse(htmlData);
+    const metaImage = htmlRootElement.querySelector(
+      'meta[property="og:image"]'
+    );
+    let thumbnailUrl = metaImage?.getAttribute("content") ?? "";
+
+    if (!thumbnailUrl.length) {
+      logger.warn(`${feedUrl}에서 썸네일 추출 실패`);
+      return thumbnailUrl;
+    }
+
+    if (!this.isUrlPath(thumbnailUrl)) {
+      thumbnailUrl = this.getHttpOriginPath(feedUrl) + thumbnailUrl;
+    }
+    return thumbnailUrl;
+  }
+
+  private isUrlPath(thumbnailUrl: string) {
+    const reg = /^(http|https):\/\//;
+    return reg.test(thumbnailUrl);
+  }
+
+  private getHttpOriginPath(feedUrl: string) {
+    return new URL(feedUrl).origin;
+  }
+
+  customUnescape(feedTitle: string): string {
+    const escapeEntity = {
+      "·": "·",
+      " ": " ",
+    };
+    Object.keys(escapeEntity).forEach((escapeKey) => {
+      const value = escapeEntity[escapeKey];
+      const regex = new RegExp(escapeKey, "g");
+      feedTitle = feedTitle.replace(regex, value);
+    });
+
+    return unescape(feedTitle);
+  }
+}

feed-crawler/src/container.ts

@@ -6,33 +6,39 @@ import { RssRepository } from './repository/rss.repository';
 import { FeedRepository } from './repository/feed.repository';
 import { RedisConnection } from './common/redis-access';
 import { TagMapRepository } from './repository/tag-map.repository';
+import { RssParser } from './common/rss-parser';
 import { ClaudeService } from './claude.service';
-import { RssParser } from './feed-crawler';
 
 container.registerSingleton<DatabaseConnection>(
   DEPENDENCY_SYMBOLS.DatabaseConnection,
-  MySQLConnection
+  MySQLConnection,
 );
+
 container.registerSingleton<RedisConnection>(
   DEPENDENCY_SYMBOLS.RedisConnection,
-  RedisConnection
+  RedisConnection,
 );
+
 container.registerSingleton<RssRepository>(
   DEPENDENCY_SYMBOLS.RssRepository,
-  RssRepository
+  RssRepository,
 );
+
 container.registerSingleton<FeedRepository>(
   DEPENDENCY_SYMBOLS.FeedRepository,
-  FeedRepository
+  FeedRepository,
 );
+
 container.registerSingleton<TagMapRepository>(
   DEPENDENCY_SYMBOLS.TagMapRepository,
-  TagMapRepository
+  TagMapRepository,
 );
+
 container.registerSingleton<ClaudeService>(
   DEPENDENCY_SYMBOLS.ClaudeService,
-  ClaudeService
+  ClaudeService,
 );
+
 container.registerSingleton<RssParser>(DEPENDENCY_SYMBOLS.RssParser, RssParser);
 
 export { container };

feed-crawler/src/feed-crawler.ts

@@ -3,13 +3,12 @@ import { RssRepository } from './repository/rss.repository';
 import logger from './common/logger';
 import { RssObj, FeedDetail, RawFeed } from './common/types';
 import { XMLParser } from 'fast-xml-parser';
-import { parse } from 'node-html-parser';
-import { unescape } from 'html-escaper';
 import {
   ONE_MINUTE,
-  INTERVAL,
+  TIME_INTERVAL,
   FEED_AI_SUMMARY_IN_PROGRESS_MESSAGE,
 } from './common/constant';
+import { RssParser } from './common/rss-parser';
 
 export class FeedCrawler {
   constructor(
@@ -25,7 +24,6 @@ export class FeedCrawler {
     await this.feedRepository.deleteRecentFeed();
 
     const rssObjects = await this.rssRepository.selectAllRss();
-
     if (!rssObjects || !rssObjects.length) {
       logger.info('등록된 RSS가 없습니다.');
       return;
@@ -39,7 +37,6 @@ export class FeedCrawler {
       return;
     }
     logger.info(`총 ${newFeeds.length}개의 새로운 피드가 있습니다.`);
-
     const insertedData: FeedDetail[] = await this.feedRepository.insertFeeds(
       newFeeds,
     );
@@ -58,9 +55,7 @@ export class FeedCrawler {
     now: number,
   ): Promise<FeedDetail[]> {
     try {
-      const TIME_INTERVAL = INTERVAL;
       const feeds = await this.fetchRss(rssObj.rssUrl);
-
       const filteredFeeds = feeds.filter((item) => {
         const pubDate = new Date(item.pubDate).setSeconds(0, 0);
         const timeDiff = (now - pubDate) / (ONE_MINUTE * TIME_INTERVAL);
@@ -146,57 +141,3 @@ export class FeedCrawler {
     }));
   }
 }
-
-export class RssParser {
-  async getThumbnailUrl(feedUrl: string) {
-    const response = await fetch(feedUrl, {
-      headers: {
-        Accept: 'text/html',
-      },
-    });
-
-    if (!response.ok) {
-      throw new Error(`${feedUrl}에 GET 요청 실패`);
-    }
-
-    const htmlData = await response.text();
-    const htmlRootElement = parse(htmlData);
-    const metaImage = htmlRootElement.querySelector(
-      'meta[property="og:image"]',
-    );
-    let thumbnailUrl = metaImage?.getAttribute('content') ?? '';
-
-    if (!thumbnailUrl.length) {
-      logger.warn(`${feedUrl}에서 썸네일 추출 실패`);
-      return thumbnailUrl;
-    }
-
-    if (!this.isUrlPath(thumbnailUrl)) {
-      thumbnailUrl = this.getHttpOriginPath(feedUrl) + thumbnailUrl;
-    }
-    return thumbnailUrl;
-  }
-
-  private isUrlPath(thumbnailUrl: string) {
-    const reg = /^(http|https):\/\//;
-    return reg.test(thumbnailUrl);
-  }
-
-  private getHttpOriginPath(feedUrl: string) {
-    return new URL(feedUrl).origin;
-  }
-
-  customUnescape(feedTitle: string): string {
-    const escapeEntity = {
-      '&middot;': '·',
-      '&nbsp;': ' ',
-    };
-    Object.keys(escapeEntity).forEach((escapeKey) => {
-      const value = escapeEntity[escapeKey];
-      const regex = new RegExp(escapeKey, 'g');
-      feedTitle = feedTitle.replace(regex, value);
-    });
-
-    return unescape(feedTitle);
-  }
-}

feed-crawler/src/main.ts

@@ -1,11 +1,13 @@
 import 'reflect-metadata';
+import './common/env-load';
 import logger from './common/logger';
-import { FeedCrawler, RssParser } from './feed-crawler';
+import { FeedCrawler } from './feed-crawler';
 import { container } from './container';
 import { RssRepository } from './repository/rss.repository';
 import { FeedRepository } from './repository/feed.repository';
 import { DEPENDENCY_SYMBOLS } from './types/dependency-symbols';
 import { DatabaseConnection } from './types/database-connection';
+import { RssParser } from './common/rss-parser';
 import { ClaudeService } from './claude.service';
 import * as schedule from 'node-schedule';
 import { RedisConnection } from './common/redis-access';