Skip to content

Commit 8ca4de1

Browse files
Jo-MinseokJo-Minseok
authored
Merge pull request #61 from boostcampwm-2024/refactor/feed-crawler-rate-limit
♻️ refactor: AI Rate Limit 해결, AI 응답 예외 처리를 위한 AI Queue 사용 및 Cron Task 추가
2 parents ceecf86 + 5e790a2 commit 8ca4de1

File tree

11 files changed

+240
-141
lines changed

11 files changed

+240
-141
lines changed

‎feed-crawler/src/claude.service.ts

Lines changed: 87 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,63 @@
11
import { injectable } from 'tsyringe';
22
import Anthropic from '@anthropic-ai/sdk';
3-
import { ClaudeResponse, FeedDetail } from './common/types';
3+
import { ClaudeResponse, FeedAIQueueItem } from './common/types';
44
import { TagMapRepository } from './repository/tag-map.repository';
55
import { FeedRepository } from './repository/feed.repository';
66
import logger from './common/logger';
7-
import { PROMPT_CONTENT } from './common/constant';
7+
import { PROMPT_CONTENT, redisConstant } from './common/constant';
8+
import { RedisConnection } from './common/redis-access';
89

910
@injectable()
1011
export class ClaudeService {
1112
private readonly client: Anthropic;
13+
private readonly nameTag: string;
1214

1315
constructor(
1416
private readonly tagMapRepository: TagMapRepository,
15-
private readonly feedRepository: FeedRepository
17+
private readonly feedRepository: FeedRepository,
18+
private readonly redisConnection: RedisConnection,
1619
) {
1720
this.client = new Anthropic({
1821
apiKey: process.env.AI_API_KEY,
1922
});
23+
this.nameTag = '[AI Service]';
2024
}
2125

22-
async useCaludeService(feeds: FeedDetail[]) {
23-
const processedFeeds = await Promise.allSettled(
26+
async startRequestAI() {
27+
const feedList: FeedAIQueueItem[] = await this.loadFeeds();
28+
const feedListWithAI = await this.requestAI(feedList);
29+
await Promise.all([
30+
this.insertTag(feedListWithAI),
31+
this.updateSummary(feedListWithAI),
32+
]);
33+
}
34+
35+
private async loadFeeds() {
36+
try {
37+
const redisSearchResult = await this.redisConnection.executePipeline(
38+
(pipeline) => {
39+
for (let i = 0; i < parseInt(process.env.AI_RATE_LIMIT_COUNT); i++) {
40+
pipeline.rpop(redisConstant.FEED_AI_QUEUE);
41+
}
42+
},
43+
);
44+
const feedObjectList: FeedAIQueueItem[] = redisSearchResult
45+
.map((result) => JSON.parse(result[1] as string))
46+
.filter((value) => value !== null);
47+
return feedObjectList;
48+
} catch (error) {
49+
logger.error(`${this.nameTag} Redis 로드한 데이터 JSON Parse 중 오류 발생:
50+
메시지: ${error.message}
51+
스택 트레이스: ${error.stack}
52+
`);
53+
}
54+
}
55+
56+
private async requestAI(feeds: FeedAIQueueItem[]) {
57+
const feedsWithAIData = await Promise.all(
2458
feeds.map(async (feed) => {
2559
try {
60+
logger.info(`${this.nameTag} AI 요청: ${JSON.stringify(feed)}`);
2661
const params: Anthropic.MessageCreateParams = {
2762
max_tokens: 8192,
2863
system: PROMPT_CONTENT,
@@ -31,92 +66,62 @@ export class ClaudeService {
3166
};
3267
const message = await this.client.messages.create(params);
3368
let responseText: string = message.content[0]['text'];
34-
responseText = responseText.replace(/\n/g, '');
35-
const result: ClaudeResponse = JSON.parse(responseText);
36-
37-
await Promise.all([
38-
this.generateTag(feed, result['tags']),
39-
this.summarize(feed, result['summary']),
40-
]);
41-
return {
42-
succeeded: true,
43-
feed,
44-
};
69+
responseText = responseText.replace(/[\n\r\t\s]+/g, ' ');
70+
logger.info(
71+
`${this.nameTag} ${feed.id} AI 요청 응답: ${responseText}`,
72+
);
73+
const responseObject: ClaudeResponse = JSON.parse(responseText);
74+
feed.summary = responseObject.summary;
75+
feed.tagList = Object.keys(responseObject.tags);
76+
return feed;
4577
} catch (error) {
4678
logger.error(
47-
`${feed.id}의 태그 생성, 컨텐츠 요약 에러 발생: `,
48-
error
79+
`${this.nameTag} ${feed.id}의 태그 생성, 컨텐츠 요약 에러 발생:
80+
메시지: ${error.message}
81+
스택 트레이스: ${error.stack}`,
4982
);
50-
return {
51-
succeeded: false,
52-
feed,
53-
};
54-
}
55-
})
56-
);
57-
58-
// TODO: Refactor
59-
const successFeeds = processedFeeds
60-
.map((result) =>
61-
result.status === 'fulfilled' && result.value.succeeded === true
62-
? result.value.feed
63-
: null
64-
)
65-
.filter((result) => result !== null);
6683

67-
// TODO: Refactor
68-
const failedFeeds = processedFeeds
69-
.map((result, index) => {
70-
if (result.status === 'rejected') {
71-
const failedFeed = feeds[index];
72-
return {
73-
succeeded: false,
74-
feed: failedFeed,
75-
};
84+
if (feed.deathCount < 3) {
85+
feed.deathCount++;
86+
this.redisConnection.rpush(redisConstant.FEED_AI_QUEUE, [
87+
JSON.stringify(feed),
88+
]);
89+
} else {
90+
logger.error(
91+
`${this.nameTag} ${feed.id}의 Death Count 3회 이상 발생 AI 요청 금지:
92+
메시지: ${error.message}
93+
스택 트레이스: ${error.stack}`,
94+
);
95+
this.feedRepository.updateNullSummary(feed.id);
96+
}
7697
}
77-
return result.status === 'fulfilled' && result.value.succeeded === false
78-
? result.value
79-
: null;
80-
})
81-
.filter((result) => result !== null && result.succeeded === false)
82-
.map((result) => result.feed);
83-
84-
logger.info(
85-
`${successFeeds.length}개의 태그 생성 및 컨텐츠 요약이 성공했습니다.\n ${failedFeeds.length}개의 태그 생성 및 컨텐츠 요약이 실패했습니다.`
98+
}),
8699
);
87-
88-
return [...successFeeds, ...failedFeeds];
100+
return feedsWithAIData.filter((value) => value !== undefined);
89101
}
90102

91-
private async generateTag(feed: FeedDetail, tags: Record<string, number>) {
92-
try {
93-
const tagList = Object.keys(tags);
94-
if (tagList.length === 0) return;
95-
await this.tagMapRepository.insertTags(feed.id, tagList);
96-
feed.tag = tagList;
97-
} catch (error) {
98-
logger.error(
99-
`[DB] 태그 데이터를 저장하는 도중 에러가 발생했습니다.
100-
에러 메시지: ${error.message}
101-
스택 트레이스: ${error.stack}`
102-
);
103-
}
104-
}
105-
106-
private async summarize(feed: FeedDetail, summary: string) {
107-
try {
108-
await this.feedRepository.insertSummary(feed.id, summary);
109-
feed.summary = summary;
110-
} catch (error) {
111-
logger.error(
112-
`[DB] 게시글 요약 데이터를 저장하는 도중 에러가 발생했습니다.
113-
에러 메시지: ${error.message}
114-
스택 트레이스: ${error.stack}`
115-
);
116-
}
103+
private insertTag(feedWithAIList: FeedAIQueueItem[]) {
104+
return feedWithAIList.map(async (feed) => {
105+
try {
106+
await this.tagMapRepository.insertTags(feed.id, feed.tagList);
107+
await this.redisConnection.hset(
108+
`feed:recent:${feed.id}`,
109+
'tag',
110+
feed.tagList.join(','),
111+
);
112+
} catch (error) {
113+
logger.error(
114+
`${this.nameTag} ${feed.id}의 태그 저장 중 에러 발생:
115+
메시지: ${error.message}
116+
스택 트레이스: ${error.stack}`,
117+
);
118+
}
119+
});
117120
}
118121

119-
public async saveAiQueue(feeds: FeedDetail[]) {
120-
await this.feedRepository.saveAiQueue(feeds);
122+
private updateSummary(feedWithAIList: FeedAIQueueItem[]) {
123+
return feedWithAIList.map((feed) =>
124+
this.feedRepository.updateSummary(feed.id, feed.summary),
125+
);
121126
}
122127
}

‎feed-crawler/src/common/constant.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ tags: { }
5757
The summary of the content should be returned in the summary field.
5858
The summary must be in Korean.
5959
When summarizing, make it engaging and intriguing so that a first-time reader would want to click on the original post.
60+
Include appropriate emojis and keep the tone light and upbeat.
6061
6162
If possible, organize the summary using Markdown format.
6263
The first line of the summary must be the title and should be displayed in **bold**.
@@ -68,6 +69,9 @@ Do not wrap the response in code blocks.
6869
Do not provide any additional explanations.
6970
Do not use any markdown formatting for the JSON output itself.
7071
72+
Important:
73+
Make sure that the last property in the JSON does not have a trailing comma.
74+
If there are multiple properties, ensure that a comma follows every property except the last one.
7175
The response should look exactly like this, without any surrounding characters:
7276
{
7377
"tags": {
@@ -80,6 +84,7 @@ The response should look exactly like this, without any surrounding characters:
8084
8185
## Do not assign any tags that are not in the predefined tag list.
8286
Strictly follow this rule.
87+
8388
Tag List:
8489
${ALLOWED_TAGS.map((tag) => `- ${tag}`).join('\n')}
8590
`;

‎feed-crawler/src/common/redis-access.ts

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ export class RedisConnection {
1515

1616
constructor() {
1717
this.nameTag = '[Redis]';
18+
this.connect();
1819
}
1920

2021
connect() {
@@ -30,14 +31,39 @@ export class RedisConnection {
3031
}
3132
}
3233

34+
async rpop(key: string) {
35+
try {
36+
return await this.redis.rpop(key);
37+
} catch (error) {
38+
logger.error(
39+
`${this.nameTag} rpop 실행 중 오류 발생:
40+
메시지: ${error.message}
41+
스택 트레이스: ${error.stack}`,
42+
);
43+
throw error;
44+
}
45+
}
46+
47+
async rpush(key: string, elements: (string | Buffer | number)[]) {
48+
try {
49+
await this.redis.rpush(key, ...elements);
50+
} catch (error) {
51+
logger.error(
52+
`${this.nameTag} rpush 실행 중 오류 발생:
53+
메시지: ${error.message}
54+
스택 트레이스: ${error.stack}`,
55+
);
56+
}
57+
}
58+
3359
async quit() {
3460
if (this.redis) {
3561
try {
3662
await this.redis.quit();
3763
} catch (error) {
3864
logger.error(
39-
`${this.nameTag} connection quit 중 오류 발생
40-
에러 메시지: ${error.message}
65+
`${this.nameTag} connection quit 중 오류 발생:
66+
메시지: ${error.message}
4167
스택 트레이스: ${error.stack}`,
4268
);
4369
}
@@ -77,4 +103,8 @@ export class RedisConnection {
77103
throw error;
78104
}
79105
}
106+
107+
async hset(key: string, ...fieldValues: (string | Buffer | number)[]) {
108+
await this.redis.hset(key, fieldValues);
109+
}
80110
}

‎feed-crawler/src/common/types.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,11 @@ export interface ClaudeResponse {
3232
tags: Record<string, number>;
3333
summary: string;
3434
}
35+
36+
export type FeedAIQueueItem = {
37+
id: number;
38+
content: string;
39+
deathCount: number;
40+
tagList?: string[];
41+
summary?: string;
42+
};

‎feed-crawler/src/feed-crawler.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import {
1010
INTERVAL,
1111
FEED_AI_SUMMARY_IN_PROGRESS_MESSAGE,
1212
} from './common/constant';
13-
import { ClaudeService } from './claude.service';
1413

1514
export class FeedCrawler {
1615
constructor(
@@ -20,6 +19,9 @@ export class FeedCrawler {
2019
) {}
2120

2221
async start() {
22+
logger.info('==========작업 시작==========');
23+
const startTime = Date.now();
24+
2325
await this.feedRepository.deleteRecentFeed();
2426

2527
const rssObjects = await this.rssRepository.selectAllRss();
@@ -43,6 +45,12 @@ export class FeedCrawler {
4345
);
4446
await this.feedRepository.saveAiQueue(insertedData);
4547
await this.feedRepository.setRecentFeedList(insertedData);
48+
49+
const endTime = Date.now();
50+
const executionTime = endTime - startTime;
51+
52+
logger.info(`실행 시간: ${executionTime / 1000}seconds`);
53+
logger.info('==========작업 완료==========');
4654
}
4755

4856
private async findNewFeeds(

0 commit comments

Comments
 (0)