Skip to content

Commit 45f8225

Browse files
committed
feat: init seeding worker
- manage it with env-var based filters - add webtorrent dependency - implement method seed to SeedingWorker class
1 parent 6c29438 commit 45f8225

File tree

6 files changed

+1010
-14
lines changed

6 files changed

+1010
-14
lines changed

docs/envs.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ This document describes the environment variables that can be used to configure
2626
| WRITE_TRANSACTION_DB_SIGNATURES | Boolean | true | If true, the transactions signatures will be written to the database. |
2727
| ENABLE_DATA_DB_WAL_CLEANUP | Boolean | false | If true, the data database WAL cleanup worker will be enabled |
2828
| MAX_DATA_ITEM_QUEUE_SIZE | Number | 100000 | Sets the maximum number of data items to queue for indexing before skipping indexing new data items |
29+
| SEEDING_WORKER_FILTER | String | undefined | The filter used to control what is seeded via the seeding worker (example via torrent protocol) |
2930
| ARNS_ROOT_HOST | String | undefined | Domain name for ArNS host |
3031
| SANDBOX_PROTOCOL | String | undefined | Protocol setting in process of creating sandbox domain in ArNS (ARNS_ROOT_HOST needs to be set for this env to have any effect) |
3132
| START_WRITERS | Boolean | true | If true, start indexing blocks, tx, ANS104 bundles |

package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
"swagger-ui-express": "^4.5.0",
4949
"umzug": "^3.2.1",
5050
"wait": "^0.4.2",
51+
"webtorrent": "^2.5.1",
5152
"winston": "^3.7.2",
5253
"yaml": "^2.3.4",
5354
"yesql": "^7.0.0"
@@ -70,6 +71,7 @@
7071
"@types/stream-json": "^1.7.2",
7172
"@types/supertest": "^2.0.16",
7273
"@types/swagger-ui-express": "^4.1.3",
74+
"@types/webtorrent": "^0.109.8",
7375
"@typescript-eslint/eslint-plugin": "^5.26.0",
7476
"@typescript-eslint/parser": "^5.26.0",
7577
"c8": "^8.0.1",

src/config.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,16 @@ export const ON_DEMAND_RETRIEVAL_ORDER = env
8989
)
9090
.split(',');
9191

92+
// seeding-worker (torrent) filter
93+
export const SEEDING_WORKER_FILTER_STRING = env.varOrUndefined(
94+
'SEEDING_WORKER_FILTER',
95+
);
96+
97+
export const SEEDING_WORKER_FILTER =
98+
SEEDING_WORKER_FILTER_STRING === undefined
99+
? undefined
100+
: createFilter(JSON.parse(SEEDING_WORKER_FILTER_STRING));
101+
92102
//
93103
// Indexing
94104
//

src/system.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ import { S3DataSource } from './data/s3-data-source.js';
8080
import { connect } from '@permaweb/aoconnect';
8181
import { DataContentAttributeImporter } from './workers/data-content-attribute-importer.js';
8282
import { SignatureFetcher } from './data/signature-fetcher.js';
83+
import { SeedingWorker } from './workers/seeding-worker.js';
8384
import { SQLiteWalCleanupWorker } from './workers/sqlite-wal-cleanup-worker.js';
8485
import { KvArnsStore } from './store/kv-arns-store.js';
8586
import { parquetExporter } from './routes/ar-io.js';
@@ -240,6 +241,16 @@ eventEmitter.on(events.TX_INDEXED, async (tx: MatchableItem) => {
240241
eventEmitter.emit(events.ANS104_TX_INDEXED, tx);
241242
eventEmitter.emit(events.ANS104_BUNDLE_INDEXED, tx);
242243
}
244+
245+
const seedingWorkerFilter = config.SEEDING_WORKER_FILTER;
246+
247+
if (
248+
seedingWorkerFilter !== undefined &&
249+
tx.id !== undefined &&
250+
(await seedingWorkerFilter.match(tx))
251+
) {
252+
seedingWorker.seed(tx.id);
253+
}
243254
});
244255

245256
eventEmitter.on(
@@ -616,6 +627,8 @@ if (dataSqliteWalCleanupWorker !== undefined) {
616627
dataSqliteWalCleanupWorker.start();
617628
}
618629

630+
export const seedingWorker = new SeedingWorker({ log, contiguousDataSource });
631+
619632
let isShuttingDown = false;
620633

621634
export const shutdown = async (express: Server) => {

src/workers/seeding-worker.ts

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/**
2+
* AR.IO Gateway
3+
* Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU Affero General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU Affero General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU Affero General Public License
16+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
*/
18+
19+
import { Logger } from 'winston';
20+
import WebTorrent from 'webtorrent';
21+
import { ContiguousDataSource } from '../types.js';
22+
23+
export class SeedingWorker {
24+
private log: Logger;
25+
private contiguousDataSource: ContiguousDataSource;
26+
27+
public webTorrentClient: WebTorrent.Instance;
28+
29+
constructor({
30+
log,
31+
contiguousDataSource,
32+
}: {
33+
log: Logger;
34+
contiguousDataSource: ContiguousDataSource;
35+
}) {
36+
this.webTorrentClient = new WebTorrent();
37+
this.contiguousDataSource = contiguousDataSource;
38+
this.log = log.child({ class: 'SeedingWorker' });
39+
}
40+
41+
async seed(txId: string) {
42+
this.log.debug(`Seeding ${txId}`);
43+
const data = await this.contiguousDataSource.getData({ id: txId });
44+
await new Promise<void>((resolve) =>
45+
this.webTorrentClient.seed(
46+
data.stream,
47+
{
48+
announce: [
49+
'wss://tracker.btorrent.xyz',
50+
'wss://tracker.openwebtorrent.com',
51+
'wss://tracker.webtorrent.io',
52+
],
53+
},
54+
(torrent: WebTorrent.Torrent) => {
55+
this.log.debug(`Seeding ${txId} started: ${torrent.magnetURI}`);
56+
resolve();
57+
},
58+
),
59+
);
60+
}
61+
}

0 commit comments

Comments
 (0)