From 2e530f58e52d4d2db541df1850fbf140c50fbca2 Mon Sep 17 00:00:00 2001 From: David Hockley Date: Sun, 19 Nov 2023 19:14:26 +0100 Subject: [PATCH] update --- README.md | 42 ++++++++++++++++++++ lib/fileManagement.js | 4 +- lib/index.d.ts | 11 ++---- lib/index.js | 90 +++++++++++++++++++++++-------------------- src/index.ts | 2 +- 5 files changed, 97 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index c693eb8..470f69c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,44 @@ # notion-parse An NPM module for downloading and Notion content and saving it as Markdown for NextJS + +I use this to download my Notion content and save it as Markdown for my NextJS blog. It's a work in progress, but it's working for me so far. + +I also use ContentLayer to make sure my fontmatter has the right fields. + +## Usage + +Here is how I use the module in my NextJS project. + +```js + +// @ts-check + +const NotionParse = require('@kodaps/notion-parse'); +const dotenv = require('dotenv'); + +dotenv.config(); + +const go = async () => { + + if (process.env.NOTION_TOKEN) { + await NotionParse.parseNotion(process.env.NOTION_TOKEN, './src/content', [ + { databaseId: process.env.NOTION_PORTFOLIO_DATABASE_ID || '', contentType: 'Portfolio' }, + { databaseId: process.env.NOTION_NEWSLETTER_DATABASE_ID || '', contentType: 'Newsletter', languageField: 'lang' }, + //{ databaseId: process.env.NOTION_PAGE_DATABASE_ID || '', contentType: 'Page', languageField: 'lang' }, + { databaseId: process.env.NOTION_POST_DATABASE_ID || '', contentType: 'Post', languageField: 'lang', filterFields: [ 'translation', 'createdAt', 'status', 'Type'] }, + { databaseId: process.env.NOTION_BITS_DATABASE_ID || '', contentType: 'Bit', languageField: 'lang' }, + ]) + } + +}; + +go().then(() => { + console.log('Done'); +}); + +``` + +This supposed several things : +1. that the files are stored in a subfolder of the folder passed in as parameter (here `./src/content`) based on the content type +2. that the ContentLayer type names map to the subfolders. So for instance for the `Post` content type, the files will be stored in `./src/content/post` +3. that the Notion token and database IDs are stored in environment variables, and that there is one database per content type \ No newline at end of file diff --git a/lib/fileManagement.js b/lib/fileManagement.js index a81bbd3..8f5bf43 100644 --- a/lib/fileManagement.js +++ b/lib/fileManagement.js @@ -2,7 +2,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.getImageFolderPath = exports.getFilePath = exports.getImageFolder = exports.getFileFolder = void 0; var getFileFolder = function (filetype, lang) { - var langBit = lang ? "".concat(lang, "/") : ''; + var langBit = (!!lang) ? "".concat(lang, "/") : ''; var contentFolder = filetype.toLowerCase(); return "./src/content/".concat(contentFolder, "/").concat(langBit); }; @@ -14,7 +14,7 @@ var getImageFolder = function (filetype) { exports.getImageFolder = getImageFolder; var getFilePath = function (slug, filetype, lang) { var fileFolder = (0, exports.getFileFolder)(filetype, lang); - return "".concat(fileFolder, "/").concat(slug, ".md"); + return "".concat(fileFolder).concat(slug, ".md"); }; exports.getFilePath = getFilePath; var getImageFolderPath = function (slug, filetype) { diff --git a/lib/index.d.ts b/lib/index.d.ts index 513f2ed..3c07ea5 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -1,15 +1,12 @@ -import { Client } from "@notionhq/client"; import { DatabaseObjectResponse, PageObjectResponse, PartialDatabaseObjectResponse, PartialPageObjectResponse } from "@notionhq/client/build/src/api-endpoints"; interface DocumentType { databaseId: string; languageField?: string; contentType: string; + filterFields?: Array; } -declare const parseNotionPage: (page: PageObjectResponse | PartialPageObjectResponse | PartialDatabaseObjectResponse | DatabaseObjectResponse, contentType: string) => Promise<{ +export declare const parseNotionPage: (page: PageObjectResponse | PartialPageObjectResponse | PartialDatabaseObjectResponse | DatabaseObjectResponse, contentType: string) => Promise<{ [key: string]: any; }>; -declare const getDatabase: (notion: Client, database_id: string, contentType: string) => Promise<{ - [key: string]: any; -}[]>; -declare const parseNotion: (token: string, contentRoot: string, contentTypes: Array) => Promise; -export { getDatabase, parseNotionPage, parseNotion, }; +export declare const parseNotion: (token: string, contentRoot: string, contentTypes: Array) => Promise; +export {}; diff --git a/lib/index.js b/lib/index.js index 5027c7b..1a9c5cb 100644 --- a/lib/index.js +++ b/lib/index.js @@ -36,17 +36,16 @@ var __generator = (this && this.__generator) || function (thisArg, body) { } }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.parseNotion = exports.parseNotionPage = exports.getDatabase = void 0; +exports.parseNotion = exports.parseNotionPage = void 0; var client_1 = require("@notionhq/client"); var notion_to_md_1 = require("notion-to-md"); var fileManagement_1 = require("./fileManagement"); -//const slugify = require('slugify'); var yaml = require('yaml'); var fs = require('fs'); var http = require('https'); var slugify = require('slugify'); -// const dotenv = require('dotenv'); var Jimp = require('jimp'); +var sleep = function (ms) { return new Promise(function (resolve) { return setTimeout(resolve, ms); }); }; var notionClient = null; var n2m = null; var setNotionSecret = function (auth) { @@ -66,9 +65,7 @@ var downloadImage = function (fileUrl, destination) { return __awaiter(void 0, v case 0: src = destination; file = './public' + src; - console.log('Checking if file exists: ', file); if (!!fs.existsSync(file)) return [3 /*break*/, 2]; - console.log("Downloading image ".concat(file)); return [4 /*yield*/, wget(fileUrl, file)]; case 1: _a.sent(); //element.properties.image.files[0].file.url, file); @@ -93,7 +90,6 @@ var manageImage = function (properties, url, contentType, name) { return __await case 0: return [4 /*yield*/, getFieldInfo(properties, 'title', contentType)]; case 1: title = _a.sent(); - console.log('Managing image for ', title, url); return [4 /*yield*/, getFieldInfo(properties, 'slug', contentType)]; case 2: slug = (_a.sent()) || slugify(title, { @@ -104,7 +100,6 @@ var manageImage = function (properties, url, contentType, name) { return __await throw new Error('No slug'); } destination = (0, fileManagement_1.getImageFolder)(contentType) + name; - console.log("Destination: ", destination); return [4 /*yield*/, downloadImage(url, destination)]; case 3: return [2 /*return*/, _a.sent()]; } @@ -112,12 +107,11 @@ var manageImage = function (properties, url, contentType, name) { return __await }); }; var getFieldInfo = function (properties, name, contentType) { return __awaiter(void 0, void 0, void 0, function () { var element, type, _a, url; - var _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m; - return __generator(this, function (_o) { - switch (_o.label) { + var _b, _c, _d, _e, _f, _g, _h, _j; + return __generator(this, function (_k) { + switch (_k.label) { case 0: element = properties[name]; - //console.log('Getting field ', name, element); if (!element) { return [2 /*return*/, null]; } @@ -131,27 +125,38 @@ var getFieldInfo = function (properties, name, contentType) { return __awaiter(v case 'checkbox': return [3 /*break*/, 5]; case 'number': return [3 /*break*/, 6]; case 'select': return [3 /*break*/, 7]; - case 'multi_select': return [3 /*break*/, 8]; - case 'files': return [3 /*break*/, 9]; + case 'created_time': return [3 /*break*/, 8]; + case 'last_edited_time': return [3 /*break*/, 9]; + case 'email': return [3 /*break*/, 10]; + case 'status': return [3 /*break*/, 11]; + case 'phone_number': return [3 /*break*/, 12]; + case 'relation': return [3 /*break*/, 13]; + case 'multi_select': return [3 /*break*/, 14]; + case 'files': return [3 /*break*/, 15]; } - return [3 /*break*/, 11]; + return [3 /*break*/, 17]; case 1: return [2 /*return*/, (_b = element.title[0]) === null || _b === void 0 ? void 0 : _b.plain_text]; case 2: return [2 /*return*/, (_c = element.rich_text[0]) === null || _c === void 0 ? void 0 : _c.plain_text]; - case 3: return [2 /*return*/, element.date.start]; + case 3: return [2 /*return*/, (_d = element.date) === null || _d === void 0 ? void 0 : _d.start]; case 4: return [2 /*return*/, element.url]; case 5: return [2 /*return*/, element.checkbox]; case 6: return [2 /*return*/, element.number]; - case 7: return [2 /*return*/, (_d = element.select) === null || _d === void 0 ? void 0 : _d.name]; - case 8: return [2 /*return*/, element.multi_select.map(function (item) { return item.name; })]; - case 9: - console.log((_e = element.files[0]) === null || _e === void 0 ? void 0 : _e.file, (_f = element.files[0]) === null || _f === void 0 ? void 0 : _f.url, (_g = element.files[0]) === null || _g === void 0 ? void 0 : _g.type, (_h = element.files[0]) === null || _h === void 0 ? void 0 : _h.name); - url = ((_j = element.files[0]) === null || _j === void 0 ? void 0 : _j.url) || ((_l = (_k = element.files[0]) === null || _k === void 0 ? void 0 : _k.file) === null || _l === void 0 ? void 0 : _l.url); + case 7: return [2 /*return*/, (_e = element.select) === null || _e === void 0 ? void 0 : _e.name]; + case 8: return [2 /*return*/, element.created_time]; + case 9: return [2 /*return*/, element.last_edited_time]; + case 10: return [2 /*return*/, element.email]; + case 11: return [2 /*return*/, element.status]; + case 12: return [2 /*return*/, element.phone_number]; + case 13: return [2 /*return*/, element.relation.map(function (item) { return item.id; })]; + case 14: return [2 /*return*/, element.multi_select.map(function (item) { return item.name; })]; + case 15: + url = ((_f = element.files[0]) === null || _f === void 0 ? void 0 : _f.url) || ((_h = (_g = element.files[0]) === null || _g === void 0 ? void 0 : _g.file) === null || _h === void 0 ? void 0 : _h.url); if (!url) { return [2 /*return*/, null]; } - return [4 /*yield*/, manageImage(properties, url, contentType, (_m = element.files[0]) === null || _m === void 0 ? void 0 : _m.name)]; - case 10: return [2 /*return*/, _o.sent()]; - case 11: throw new Error("Unknown type ".concat(type)); + return [4 /*yield*/, manageImage(properties, url, contentType, (_j = element.files[0]) === null || _j === void 0 ? void 0 : _j.name)]; + case 16: return [2 /*return*/, _k.sent()]; + case 17: throw new Error("Unknown type ".concat(type)); } }); }); }; @@ -165,11 +170,10 @@ function wget(url, dest) { wget(String(response.headers.location), dest); } else { - console.log('Dpwnloading', url, 'to', dest); + console.log('Downloading', url, 'to', dest); var file_1 = fs.createWriteStream(dest); response.pipe(file_1); file_1.on('finish', function () { - console.log('Download done'); file_1.close(); res(); }); @@ -215,7 +219,7 @@ var parseNotionPage = function (page, contentType) { return __awaiter(void 0, vo exports.parseNotionPage = parseNotionPage; var checkFolder = function (dir) { if (!fs.existsSync(dir)) { - fs.mkdirSync(dir); + fs.mkdirSync(dir, { recursive: true }); } }; var getDatabase = function (notion, database_id, contentType) { return __awaiter(void 0, void 0, void 0, function () { @@ -234,8 +238,7 @@ var getDatabase = function (notion, database_id, contentType) { return __awaiter case 2: if (!(_i < results_1.length)) return [3 /*break*/, 5]; page = results_1[_i]; - console.log('page: ', page); - return [4 /*yield*/, parseNotionPage(page, contentType)]; + return [4 /*yield*/, (0, exports.parseNotionPage)(page, contentType)]; case 3: item = _a.sent(); ret.push(item); @@ -247,7 +250,6 @@ var getDatabase = function (notion, database_id, contentType) { return __awaiter } }); }); }; -exports.getDatabase = getDatabase; var saveFile = function (frontMatter, type, languageField) { return __awaiter(void 0, void 0, void 0, function () { var notionId, lang, title, slug, mdblocks, imageBlocks, images, imagePath, _i, imageBlocks_1, block, data, url, name_1, filename, src, file, mdBody, _a, images_1, image, newFile; return __generator(this, function (_b) { @@ -259,7 +261,6 @@ var saveFile = function (frontMatter, type, languageField) { return __awaiter(vo notionId = frontMatter['notionId']; lang = languageField ? frontMatter[languageField] : ''; title = frontMatter['title']; - console.log('Processing page: ', title); if (!title && !frontMatter['slug']) { throw new Error("No title or slug in front matter for ".concat(notionId, " of type ").concat(type)); } @@ -267,6 +268,7 @@ var saveFile = function (frontMatter, type, languageField) { return __awaiter(vo lower: true, strict: true, }); + frontMatter['slug'] = slug; return [4 /*yield*/, n2m.pageToMarkdown(notionId)]; case 1: mdblocks = _b.sent(); @@ -311,12 +313,11 @@ var saveFile = function (frontMatter, type, languageField) { return __awaiter(vo mdBody.parent = mdBody.parent.replace(image.url, image.src); } newFile = (0, fileManagement_1.getFilePath)(slug, type, lang); - console.log('preparing to save: ', newFile); try { fs.writeFileSync(newFile, toFrontMatter(frontMatter) + mdBody.parent); } catch (e) { - console.log('error title: ', title); + console.log('error with file: ', newFile); console.error(e); } return [2 /*return*/]; @@ -324,22 +325,20 @@ var saveFile = function (frontMatter, type, languageField) { return __awaiter(vo }); }); }; var parseNotion = function (token, contentRoot, contentTypes) { return __awaiter(void 0, void 0, void 0, function () { - var _i, contentTypes_1, type, databaseId, lang, contentType, database, _a, database_1, page; - return __generator(this, function (_b) { - switch (_b.label) { + var _i, contentTypes_1, type, databaseId, lang, contentType, database, _a, database_1, page, _b, _c, field; + return __generator(this, function (_d) { + switch (_d.label) { case 0: setNotionSecret(token); addDocumentTypes(contentTypes); - console.log('Parsing notion content: ', contentRoot, contentTypes); if (!notionClient) { throw new Error('Notion client incorretly setup'); } _i = 0, contentTypes_1 = contentTypes; - _b.label = 1; + _d.label = 1; case 1: if (!(_i < contentTypes_1.length)) return [3 /*break*/, 7]; type = contentTypes_1[_i]; - console.log('Processing type: ', type); databaseId = type.databaseId; lang = type.languageField; contentType = type.contentType || databaseId; @@ -351,16 +350,23 @@ var parseNotion = function (token, contentRoot, contentTypes) { return __awaiter } return [4 /*yield*/, getDatabase(notionClient, databaseId, contentType)]; case 2: - database = _b.sent(); + database = _d.sent(); _a = 0, database_1 = database; - _b.label = 3; + _d.label = 3; case 3: if (!(_a < database_1.length)) return [3 /*break*/, 6]; page = database_1[_a]; + sleep(400); + for (_b = 0, _c = (type.filterFields || []); _b < _c.length; _b++) { + field = _c[_b]; + if (page[field]) { + delete page[field]; + } + } return [4 /*yield*/, saveFile(page, contentType, lang)]; case 4: - _b.sent(); - _b.label = 5; + _d.sent(); + _d.label = 5; case 5: _a++; return [3 /*break*/, 3]; diff --git a/src/index.ts b/src/index.ts index 412b55c..1acc583 100644 --- a/src/index.ts +++ b/src/index.ts @@ -179,7 +179,7 @@ export const parseNotionPage = async (page:PageObjectResponse| PartialPageObject const checkFolder = (dir: string) => { if (!fs.existsSync(dir)){ - fs.mkdirSync(dir); + fs.mkdirSync(dir, { recursive: true }); } }