Paperless ngx data connector

collector/extensions/index.js

@@ -201,7 +201,26 @@ function extensions(app) {
       return;
     }
   );
-}
 
+  app.post(
+    "/ext/paperless-ngx",
+    [verifyPayloadIntegrity, setDataSigner],
+    async function (request, response) {
+      try {
+        const { loadPaperlessNgx } = require("../utils/extensions/PaperlessNgx");
+        const result = await loadPaperlessNgx(reqBody(request), response);
+        response.status(200).json(result);
+      } catch (e) {
+        console.error(e);
+        response.status(400).json({
+          success: false,
+          reason: e.message,
+          data: null,
+        });
+      }
+      return;
+    }
+  );
+}
 
 module.exports = extensions;

collector/extensions/resync/index.js

@@ -144,10 +144,40 @@ async function resyncDrupalWiki({ chunkSource }, response) {
   }
 }
 
+/**
+ * Fetches the content of a specific Paperless-ngx document via its chunkSource.
+ * Returns the content as a text string of the document.
+ * @param {object} data - metadata from document (eg: chunkSource)
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncPaperlessNgx({ chunkSource }, response) {
+  if (!chunkSource) throw new Error('Invalid source property provided');
+  try {
+    const source = response.locals.encryptionWorker.expandPayload(chunkSource);
+    const { PaperlessNgxLoader } = require("../../utils/extensions/PaperlessNgx/PaperlessNgxLoader");
+    const loader = new PaperlessNgxLoader({
+      baseUrl: source.searchParams.get('baseUrl'),
+      apiToken: source.searchParams.get('token'),
+    });
+    const documentId = source.pathname.split('//')[1];
+    const content = await loader.fetchDocumentContent(documentId);
+
+    if (!content) throw new Error('Failed to fetch document content');
+    response.status(200).json({ success: true, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
 module.exports = {
   link: resyncLink,
   youtube: resyncYouTube,
   confluence: resyncConfluence,
   github: resyncGithub,
   drupalwiki: resyncDrupalWiki,
+  "paperless-ngx": resyncPaperlessNgx,
 }

collector/utils/extensions/PaperlessNgx/PaperlessNgxLoader/index.js

@@ -0,0 +1,126 @@
+const { htmlToText } = require("html-to-text");
+const pdf = require("pdf-parse");
+
+class PaperlessNgxLoader {
+  constructor({ baseUrl, apiToken }) {
+    this.baseUrl = baseUrl.replace(/\/$/, "");
+    this.apiToken = apiToken;
+  }
+
+  async load() {
+    try {
+      const documents = await this.fetchAllDocuments();
+      return documents.map((doc) => this.createDocumentFromPage(doc));
+    } catch (error) {
+      console.error("Error:", error);
+      throw error;
+    }
+  }
+
+  async fetchAllDocuments() {
+    try {
+      const response = await fetch(`${this.baseUrl}/api/documents/`, {
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Token ${this.apiToken}`,
+        },
+      });
+
+      if (!response.ok) {
+        throw new Error(
+          `Failed to fetch documents from Paperless-ngx: ${response.status}`
+        );
+      }
+
+      const data = await response.json();
+      const documents = data.results || [];
+
+      const documentsWithContent = await Promise.all(
+        documents.map(async (doc) => {
+          const content = await this.fetchDocumentContent(doc.id);
+          return { ...doc, content };
+        })
+      );
+
+      return documentsWithContent;
+    } catch (error) {
+      throw new Error(
+        `Failed to fetch documents from Paperless-ngx: ${error.message}`
+      );
+    }
+  }
+
+  async fetchDocumentContent(documentId) {
+    try {
+      const response = await fetch(
+        `${this.baseUrl}/api/documents/${documentId}/download/`,
+        {
+          headers: {
+            Authorization: `Token ${this.apiToken}`,
+          },
+        }
+      );
+
+      if (!response.ok) {
+        throw new Error(`Failed to fetch document content: ${response.status}`);
+      }
+
+      // Process text and pdf files
+      const contentType = response.headers.get("content-type");
+      let content;
+
+      if (contentType.includes("text/plain")) {
+        content = await response.text();
+      } else if (contentType.includes("application/pdf")) {
+        const buffer = await response.arrayBuffer();
+        content = await this.parsePdfContent(buffer);
+      } else {
+        // Fallback to text content
+        content = await response.text();
+      }
+
+      return content;
+    } catch (error) {
+      console.error(
+        `Failed to fetch content for document ${documentId}:`,
+        error
+      );
+      return "";
+    }
+  }
+
+  async parsePdfContent(buffer) {
+    try {
+      const data = await pdf(Buffer.from(buffer));
+      return data.text;
+    } catch (error) {
+      console.error("Failed to parse PDF content:", error);
+      return "";
+    }
+  }
+
+  createDocumentFromPage(doc) {
+    const content = doc.content || "";
+    const plainTextContent = htmlToText(content, {
+      wordwrap: false,
+      preserveNewlines: true,
+    });
+
+    return {
+      pageContent: plainTextContent,
+      metadata: {
+        id: doc.id,
+        title: doc.title,
+        created: doc.created,
+        modified: doc.modified,
+        added: doc.added,
+        tags: doc.tags,
+        correspondent: doc.correspondent,
+        documentType: doc.document_type,
+        url: `${this.baseUrl}/documents/${doc.id}`,
+      },
+    };
+  }
+}
+
+module.exports = PaperlessNgxLoader;

collector/utils/extensions/PaperlessNgx/index.js

@@ -0,0 +1,131 @@
+const fs = require("fs");
+const path = require("path");
+const { default: slugify } = require("slugify");
+const { v4 } = require("uuid");
+const { writeToServerDocuments, sanitizeFileName } = require("../../files");
+const { tokenizeString } = require("../../tokenizer");
+const { validBaseUrl } = require("../../http");
+const PaperlessNgxLoader = require("./PaperlessNgxLoader");
+
+/**
+ * Load documents from a Paperless-ngx instance
+ * @param {object} args - forwarded request body params
+ * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker
+ * @returns
+ */
+async function loadPaperlessNgx({ baseUrl = null, apiToken = null }, response) {
+  if (!baseUrl || !validBaseUrl(baseUrl)) {
+    return {
+      success: false,
+      reason: "Provided base URL is not a valid URL.",
+    };
+  }
+
+  if (!apiToken) {
+    return {
+      success: false,
+      reason:
+        "You need to provide an API token to use the Paperless-ngx connector.",
+    };
+  }
+
+  const { origin, hostname } = new URL(baseUrl);
+  console.log(`-- Working Paperless-ngx ${origin} --`);
+  const loader = new PaperlessNgxLoader({
+    baseUrl: origin,
+    apiToken,
+  });
+
+  const { docs, error } = await loader
+    .load()
+    .then((docs) => {
+      return { docs, error: null };
+    })
+    .catch((e) => {
+      return {
+        docs: [],
+        error: e.message?.split("Error:")?.[1] || e.message,
+      };
+    });
+
+  if (!docs.length || !!error) {
+    return {
+      success: false,
+      reason: error ?? "No documents found in that Paperless-ngx instance.",
+    };
+  }
+
+  const outFolder = slugify(
+    `paperless-${hostname}-${v4().slice(0, 4)}`
+  ).toLowerCase();
+
+  const outFolderPath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(
+          __dirname,
+          `../../../../server/storage/documents/${outFolder}`
+        )
+      : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`);
+
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
+
+  docs.forEach((doc) => {
+    if (!doc.pageContent) return;
+
+    const data = {
+      id: v4(),
+      url: doc.metadata.url,
+      title: doc.metadata.title,
+      docAuthor: doc.metadata.correspondent || "Unknown",
+      description: `Document Type: ${doc.metadata.documentType || "Unknown"}`,
+      docSource: `${origin} Paperless-ngx`,
+      chunkSource: generateChunkSource(
+        { doc, baseUrl: origin, apiToken },
+        response.locals.encryptionWorker
+      ),
+      published: doc.metadata.created,
+      wordCount: doc.pageContent.split(" ").length,
+      pageContent: doc.pageContent,
+      token_count_estimate: tokenizeString(doc.pageContent),
+    };
+
+    console.log(
+      `[Paperless-ngx Loader]: Saving ${doc.metadata.title} to ${outFolder}`
+    );
+
+    const fileName = sanitizeFileName(
+      `${slugify(doc.metadata.title)}-${data.id}`
+    );
+    writeToServerDocuments(data, fileName, outFolderPath);
+  });
+
+  return {
+    success: true,
+    reason: null,
+    data: {
+      files: docs.length,
+      destination: outFolder,
+    },
+  };
+}
+
+/**
+ * Generate the full chunkSource for a specific Paperless-ngx document so that we can resync it later.
+ * @param {object} chunkSourceInformation
+ * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker
+ * @returns {string}
+ */
+function generateChunkSource({ doc, baseUrl, apiToken }, encryptionWorker) {
+  const payload = {
+    baseUrl,
+    token: apiToken,
+  };
+  return `paperless-ngx://${doc.metadata.id}?payload=${encryptionWorker.encrypt(
+    JSON.stringify(payload)
+  )}`;
+}
+
+module.exports = {
+  loadPaperlessNgx,
+};

frontend/src/components/DataConnectorOption/media/index.js

@@ -5,6 +5,7 @@ import Link from "./link.svg";
 import Confluence from "./confluence.jpeg";
 import DrupalWiki from "./drupalwiki.jpg";
 import Obsidian from "./obsidian.png";
+import PaperlessNgx from "./paperless-ngx.jpeg";
 
 const ConnectorImages = {
   github: GitHub,
@@ -14,6 +15,7 @@ const ConnectorImages = {
   confluence: Confluence,
   drupalwiki: DrupalWiki,
   obsidian: Obsidian,
+  paperlessNgx: PaperlessNgx,
 };
 
 export default ConnectorImages;